From b8920b9dab0d583a77c38b5b792f875fffaa6f47 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 14 Mar 2018 17:03:04 -0400 Subject: [PATCH 01/61] wip: remove quotes from schema/table names --- dbt/adapters/default.py | 28 ++++++++++++------- dbt/adapters/postgres.py | 18 ++++++------ dbt/adapters/snowflake.py | 21 +++++++------- .../global_project/macros/adapters/common.sql | 8 +++--- .../macros/adapters/postgres.sql | 2 +- .../macros/adapters/redshift.sql | 12 ++++---- .../macros/adapters/snowflake.sql | 4 +-- .../macros/materializations/archive.sql | 18 ++++++------ .../macros/materializations/incremental.sql | 8 +++--- .../macros/materializations/table.sql | 2 +- dbt/utils.py | 19 ++----------- 11 files changed, 67 insertions(+), 73 deletions(-) diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index 1f6cd8968c0..33846a097a7 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -1,5 +1,4 @@ import copy -import itertools import multiprocessing import time import agate @@ -35,6 +34,7 @@ class DefaultAdapter(object): "add_query", "expand_target_column_types", "quote_schema_and_table", + "get_schema_and_table", "execute" ] @@ -140,7 +140,7 @@ def drop(cls, profile, schema, relation, relation_type, model_name=None): @classmethod def drop_relation(cls, profile, schema, rel_name, rel_type, model_name): - relation = cls.quote_schema_and_table(profile, schema, rel_name) + relation = cls.get_schema_and_table(profile, schema, rel_name) sql = 'drop {} if exists {} cascade'.format(rel_type, relation) connection, cursor = cls.add_query(profile, sql, model_name) @@ -155,16 +155,15 @@ def drop_table(cls, profile, schema, table, model_name): @classmethod def truncate(cls, profile, schema, table, model_name=None): - relation = cls.quote_schema_and_table(profile, schema, table) + relation = cls.get_schema_and_table(profile, schema, table) sql = 'truncate table {}'.format(relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod def rename(cls, profile, schema, from_name, to_name, model_name=None): - from_relation = cls.quote_schema_and_table(profile, schema, from_name) - to_relation = cls.quote(to_name) - sql = 'alter table {} rename to {}'.format(from_relation, to_relation) + from_relation = cls.get_schema_and_table(profile, schema, from_name) + sql = 'alter table {} rename to {}'.format(from_relation, to_name) connection, cursor = cls.add_query(profile, sql, model_name) @@ -196,11 +195,11 @@ def _get_columns_in_table_sql(cls, schema_name, table_name): sql = """ select column_name, data_type, character_maximum_length from information_schema.columns - where table_name = '{table_name}' + where table_name ilike '{table_name}' """.format(table_name=table_name).strip() if schema_name is not None: - sql += (" AND table_schema = '{schema_name}'" + sql += (" AND table_schema ilike '{schema_name}'" .format(schema_name=schema_name)) return sql @@ -261,12 +260,12 @@ def expand_target_column_types(cls, profile, ### @classmethod def get_create_schema_sql(cls, schema): - return ('create schema if not exists "{schema}"' + return ('create schema if not exists {schema}' .format(schema=schema)) @classmethod def get_drop_schema_sql(cls, schema): - return ('drop schema if exists "{schema} cascade"' + return ('drop schema if exists {schema} cascade' .format(schema=schema)) ### @@ -632,6 +631,15 @@ def quote_schema_and_table(cls, profile, schema, table, model_name=None): return '{}.{}'.format(cls.quote(schema), cls.quote(table)) + @classmethod + def get_schema_and_table(cls, profile, schema, table, quote=False, + model_name=None): + if quote: + return cls.quote_schema_and_table(profile, schema, table, + model_name) + else: + return '{}.{}'.format(schema, table) + @classmethod def handle_csv_table(cls, profile, schema, table_name, agate_table, full_refresh=False): diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres.py index 6169afdcabb..2cd35cec92a 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres.py @@ -97,10 +97,10 @@ def alter_column_type(cls, profile, schema, table, column_name, } sql = """ - alter table "{schema}"."{table}" add column "{tmp_column}" {dtype}; - update "{schema}"."{table}" set "{tmp_column}" = "{old_column}"; - alter table "{schema}"."{table}" drop column "{old_column}" cascade; - alter table "{schema}"."{table}" rename column "{tmp_column}" to "{old_column}"; + alter table {schema}.{table} add column "{tmp_column}" {dtype}; + update {schema}.{table} set "{tmp_column}" = "{old_column}"; + alter table {schema}.{table} drop column "{old_column}" cascade; + alter table {schema}.{table} rename column "{tmp_column}" to "{old_column}"; """.format(**opts).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name) @@ -198,8 +198,9 @@ def create_csv_table(cls, profile, schema, table_name, agate_table): for idx, col_name in enumerate(agate_table.column_names): type_ = cls.convert_agate_type(agate_table, idx) col_sqls.append('{} {}'.format(col_name, type_)) - sql = 'create table "{}"."{}" ({})'.format(schema, table_name, - ", ".join(col_sqls)) + + relation = cls.get_schema_and_table(profile, schema, table_name) + sql = 'create table {} ({})'.format(relation, ", ".join(col_sqls)) return cls.add_query(profile, sql) @classmethod @@ -222,9 +223,8 @@ def load_csv_rows(cls, profile, schema, table_name, agate_table): placeholders.append("({})".format( ", ".join("%s" for _ in agate_table.column_names))) - sql = ('insert into {}.{} ({}) values {}' - .format(cls.quote(schema), - cls.quote(table_name), + sql = ('insert into {} ({}) values {}' + .format(cls.get_schema_and_table(schema, table_name), cols_sql, ",\n".join(placeholders))) diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake.py index 6afa9d06327..437dd4a2805 100644 --- a/dbt/adapters/snowflake.py +++ b/dbt/adapters/snowflake.py @@ -107,12 +107,13 @@ def query_for_existing(cls, profile, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): schemas = [schemas] - schema_list = ",".join(["'{}'".format(schema) for schema in schemas]) + schemas_upper = ["'{}'".format(schema.upper()) for schema in schemas] + schema_list = ",".join(schemas_upper) sql = """ - select TABLE_NAME as name, TABLE_TYPE as type - from INFORMATION_SCHEMA.TABLES - where TABLE_SCHEMA in ({schema_list}) + select table_name as name, table_type as type + from information_schema.tables + where upper(table_schema) in ({schema_list}) """.format(schema_list=schema_list).strip() # noqa _, cursor = cls.add_query(profile, sql, model_name, auto_begin=False) @@ -130,8 +131,8 @@ def query_for_existing(cls, profile, schemas, model_name=None): @classmethod def rename(cls, profile, schema, from_name, to_name, model_name=None): - sql = (('alter table "{schema}"."{from_name}" ' - 'rename to "{schema}"."{to_name}"') + sql = (('alter table {schema}.{from_name} ' + 'rename to {schema}.{to_name}') .format(schema=schema, from_name=from_name, to_name=to_name)) @@ -155,7 +156,7 @@ def create_schema(cls, profile, schema, model_name=None): @classmethod def get_existing_schemas(cls, profile, model_name=None): - sql = "select distinct SCHEMA_NAME from INFORMATION_SCHEMA.SCHEMATA" + sql = "select distinct schema_name from information_schema.schemata" connection, cursor = cls.add_query(profile, sql, model_name, select_schema=False, @@ -168,9 +169,9 @@ def get_existing_schemas(cls, profile, model_name=None): def check_schema_exists(cls, profile, schema, model_name=None): sql = """ select count(*) - from INFORMATION_SCHEMA.SCHEMATA - where SCHEMA_NAME = '{schema}' - """.format(schema=schema).strip() # noqa + from information_schema.schemata + where upper(schema_name) = '{schema}' + """.format(schema=schema.upper()).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name, select_schema=False, diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 9d02ca32315..b8138b3e62c 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -29,7 +29,7 @@ {%- endmacro %} {% macro create_schema(schema_name) %} - create schema if not exists "{{ schema_name }}"; + create schema if not exists {{ schema_name }}; {% endmacro %} @@ -38,7 +38,7 @@ {%- endmacro %} {% macro default__create_table_as(temporary, identifier, sql) -%} - create {% if temporary: -%}temporary{%- endif %} table "{{ schema }}"."{{ identifier }}" as ( + create {% if temporary: -%}temporary{%- endif %} table {{ schema }}.{{ identifier }} as ( {{ sql }} ); {% endmacro %} @@ -48,7 +48,7 @@ {%- endmacro %} {% macro default__create_view_as(identifier, sql) -%} - create view "{{ schema }}"."{{ identifier }}" as ( + create view {{ schema }}.{{ identifier }} as ( {{ sql }} ); {% endmacro %} @@ -59,7 +59,7 @@ {%- endmacro %} {% macro default__create_archive_table(schema, identifier, columns) -%} - create table if not exists "{{ schema }}"."{{ identifier }}" ( + create table if not exists {{ schema }}.{{ identifier }} ( {{ column_list_for_create_table(columns) }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/postgres.sql b/dbt/include/global_project/macros/adapters/postgres.sql index 2e945b17516..166c27e6b6a 100644 --- a/dbt/include/global_project/macros/adapters/postgres.sql +++ b/dbt/include/global_project/macros/adapters/postgres.sql @@ -1,6 +1,6 @@ {% macro postgres__create_table_as(temporary, identifier, sql) -%} create {% if temporary: -%}temporary{%- endif %} table - {% if not temporary: -%}"{{ schema }}".{%- endif %}"{{ identifier }}" as ( + {% if not temporary: -%}{{ schema }}.{%- endif %}{{ identifier }} as ( {{ sql }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/redshift.sql b/dbt/include/global_project/macros/adapters/redshift.sql index 6fc15111455..9aae8a341f3 100644 --- a/dbt/include/global_project/macros/adapters/redshift.sql +++ b/dbt/include/global_project/macros/adapters/redshift.sql @@ -5,7 +5,7 @@ {%- if dist in ['all', 'even'] -%} diststyle {{ dist }} {%- else -%} - diststyle key distkey ("{{ dist }}") + diststyle key distkey ({{ dist }}) {%- endif -%} {%- endif -%} @@ -19,7 +19,7 @@ {%- set sort = [sort] -%} {%- endif -%} {%- for item in sort -%} - "{{ item }}" + {{ item }} {%- if not loop.last -%},{%- endif -%} {%- endfor -%} ) @@ -38,9 +38,9 @@ validator=validation.any[list, basestring]) -%} {% if temporary %} - {% set relation = adapter.quote(identifier) %} + {% set relation = identifier %} {% else %} - {% set relation = adapter.quote(schema) ~ '.' ~ adapter.quote(identifier) %} + {% set relation = schema ~ '.' ~ identifier %} {% endif %} create {% if temporary -%}temporary{%- endif %} table {{ relation }} @@ -56,14 +56,14 @@ {% set bind_qualifier = '' if config.get('bind', default=True) else 'with no schema binding' %} - create view "{{ schema }}"."{{ identifier }}" as ( + create view {{ schema }}.{{ identifier }} as ( {{ sql }} ) {{ bind_qualifier }}; {% endmacro %} {% macro redshift__create_archive_table(schema, identifier, columns) -%} - create table if not exists "{{ schema }}"."{{ identifier }}" ( + create table if not exists {{ schema }}.{{ identifier }} ( {{ column_list_for_create_table(columns) }} ) {{ dist('dbt_updated_at') }} diff --git a/dbt/include/global_project/macros/adapters/snowflake.sql b/dbt/include/global_project/macros/adapters/snowflake.sql index 28b876f6569..716f67a07dc 100644 --- a/dbt/include/global_project/macros/adapters/snowflake.sql +++ b/dbt/include/global_project/macros/adapters/snowflake.sql @@ -1,10 +1,10 @@ {% macro snowflake__create_table_as(temporary, identifier, sql) -%} {% if temporary %} - use schema "{{ schema }}"; + use schema {{ schema }}; {% endif %} create {% if temporary: -%}temporary{%- endif %} table - {% if not temporary: -%}"{{ schema }}".{%- endif %}"{{ identifier }}" as ( + {% if not temporary: -%}{{ schema }}.{%- endif %}{{ identifier }} as ( {{ sql }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/archive.sql b/dbt/include/global_project/macros/materializations/archive.sql index f9ba53c431f..68747b0ffe3 100644 --- a/dbt/include/global_project/macros/materializations/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive.sql @@ -1,6 +1,6 @@ {% macro archive_select(source_schema, source_table, target_schema, target_table, unique_key, updated_at) %} - with "current_data" as ( + with current_data as ( select {% for col in adapter.get_columns_in_table(source_schema, source_table) %} @@ -10,7 +10,7 @@ {{ unique_key }} as "dbt_pk", {{ updated_at }} as "valid_from", null::timestamp as "tmp_valid_to" - from "{{ source_schema }}"."{{ source_table }}" + from {{ source_schema }}.{{ source_table }} ), @@ -24,7 +24,7 @@ {{ unique_key }} as "dbt_pk", "valid_from", "valid_to" as "tmp_valid_to" - from "{{ target_schema }}"."{{ target_table }}" + from {{ target_schema }}.{{ target_table }} ), @@ -104,7 +104,7 @@ {% for col in missing_columns %} {% call statement() %} - alter table "{{ target_schema }}"."{{ target_table }}" add column "{{ col.name }}" {{ col.data_type }}; + alter table {{ target_schema }}.{{ target_table }} add column "{{ col.name }}" {{ col.data_type }}; {% endcall %} {% endfor %} @@ -130,15 +130,15 @@ to_table=target_table) }} {% call statement('main') -%} - update "{{ target_schema }}"."{{ identifier }}" set "valid_to" = "tmp"."valid_to" - from "{{ tmp_identifier }}" as "tmp" - where "tmp"."scd_id" = "{{ target_schema }}"."{{ identifier }}"."scd_id" + update {{ target_schema }}.{{ identifier }} set "valid_to" = "tmp"."valid_to" + from {{ tmp_identifier }} as "tmp" + where "tmp"."scd_id" = {{ target_schema }}.{{ identifier }}."scd_id" and "change_type" = 'update'; - insert into "{{ target_schema }}"."{{ identifier }}" ( + insert into {{ target_schema }}.{{ identifier }} ( {{ column_list(dest_columns) }} ) - select {{ column_list(dest_columns) }} from "{{ tmp_identifier }}" + select {{ column_list(dest_columns) }} from {{ tmp_identifier }} where "change_type" = 'insert'; {% endcall %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index e6f33710c1d..f82f2112fc4 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -4,10 +4,10 @@ {%- set identifier = model['name'] -%} delete - from "{{ schema }}"."{{ identifier }}" + from {{ schema }}.{{ identifier }} where ({{ unique_key }}) in ( select ({{ unique_key }}) - from "{{ identifier }}__dbt_incremental_tmp" + from {{ identifier }}__dbt_incremental_tmp ); {%- endmacro %} @@ -78,10 +78,10 @@ {%- endif %} - insert into "{{ schema }}"."{{ identifier }}" ({{ dest_cols_csv }}) + insert into {{ schema }}.{{ identifier }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from "{{ identifier }}__dbt_incremental_tmp" + from {{ identifier }}__dbt_incremental_tmp ); {% endcall %} {%- endif %} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 96691afdbec..0dfecce3280 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -32,7 +32,7 @@ insert into {{ schema }}.{{ identifier }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from "{{ tmp_identifier }}" + from {{ tmp_identifier }} ); {%- else -%} {{ create_table_as(False, identifier, sql) }} diff --git a/dbt/utils.py b/dbt/utils.py index cabcfe427b2..2ebd0b76575 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -50,21 +50,6 @@ def __init__(self, profile, adapter, node, use_temp=False): self.materialized = get_materialization(node) self.sql = node.get('injected_sql') - self.do_quote = self._get_quote_function(profile, adapter) - - def _get_quote_function(self, profile, adapter): - - # make a closure so we don't need to store the profile - # on the `Relation` object. That shouldn't be accessible in user-land - def quote(schema, table): - return adapter.quote_schema_and_table( - profile=profile, - schema=schema, - table=table - ) - - return quote - def _get_table_name(self, node): return model_immediate_name(node, dbt.flags.NON_DESTRUCTIVE) @@ -73,13 +58,13 @@ def final_name(self): msg = "final_name() was called on an ephemeral model" dbt.exceptions.raise_compiler_error(msg, self.node) else: - return self.do_quote(self.schema, self.name) + return "{}.{}".format(self.schema, self.table) def __repr__(self): if self.materialized == 'ephemeral': return '__dbt__CTE__{}'.format(self.name) else: - return self.do_quote(self.schema, self.table) + return "{}.{}".format(self.schema, self.table) def coalesce(*args): From d6f58b517417088cc6e13cb5e12f57403d6d6988 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 15 Mar 2018 12:55:42 -0400 Subject: [PATCH 02/61] update for new integration project model names --- .../models/view_summary.sql | 2 +- .../test_simple_dependency.py | 24 +++++++++---------- .../test_simple_dependency_with_configs.py | 24 +++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/test/integration/006_simple_dependency_test/models/view_summary.sql b/test/integration/006_simple_dependency_test/models/view_summary.sql index 64a6ac67f0e..4ff38b12de7 100644 --- a/test/integration/006_simple_dependency_test/models/view_summary.sql +++ b/test/integration/006_simple_dependency_test/models/view_summary.sql @@ -8,7 +8,7 @@ with t as ( - select * from {{ ref('view') }} + select * from {{ ref('view_model') }} ) diff --git a/test/integration/006_simple_dependency_test/test_simple_dependency.py b/test/integration/006_simple_dependency_test/test_simple_dependency.py index 3ae44ee7422..e45501976ef 100644 --- a/test/integration/006_simple_dependency_test/test_simple_dependency.py +++ b/test/integration/006_simple_dependency_test/test_simple_dependency.py @@ -28,8 +28,8 @@ def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) - self.assertTablesEqual("seed","table") - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","table_model") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed_summary","view_summary") @@ -39,24 +39,24 @@ def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) - self.assertTablesEqual("seed","table") - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","table_model") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") @attr(type='postgres') def test_simple_dependency_with_models(self): self.run_dbt(["deps"]) - self.run_dbt(["run", '--models', 'view+']) + self.run_dbt(["run", '--models', 'view_model+']) - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed_summary","view_summary") created_models = self.get_models_in_schema() - self.assertFalse('table' in created_models) + self.assertFalse('table_model' in created_models) self.assertFalse('incremental' in created_models) - self.assertEqual(created_models['view'], 'view') + self.assertEqual(created_models['view_model'], 'view') self.assertEqual(created_models['view_summary'], 'view') class TestSimpleDependencyBranch(DBTIntegrationTest): @@ -85,14 +85,14 @@ def deps_run_assert_equality(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) - self.assertTablesEqual("seed","table") - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","table_model") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") created_models = self.get_models_in_schema() - self.assertEqual(created_models['table'], 'table') - self.assertEqual(created_models['view'], 'view') + self.assertEqual(created_models['table_model'], 'table') + self.assertEqual(created_models['view_model'], 'view') self.assertEqual(created_models['view_summary'], 'view') self.assertEqual(created_models['incremental'], 'table') diff --git a/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py b/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py index ac0744ed74f..65627ad72da 100644 --- a/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py +++ b/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py @@ -39,8 +39,8 @@ def test_simple_dependency(self): self.run_dbt(["run"]) self.assertTablesEqual('seed_config_expected_1',"config") - self.assertTablesEqual("seed","table") - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","table_model") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") @@ -73,8 +73,8 @@ def test_simple_dependency(self): self.run_dbt(["run"]) self.assertTablesEqual('seed_config_expected_2',"config") - self.assertTablesEqual("seed","table") - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","table_model") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") @@ -111,8 +111,8 @@ def test_simple_dependency(self): self.run_dbt(["run"]) self.assertTablesEqual('seed_config_expected_3',"config") - self.assertTablesEqual("seed","table") - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","table_model") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") @@ -134,11 +134,11 @@ def project_config(self): } }, # disable the table model - "table": { + "table_model": { "enabled": False, }, # override materialization settings - "view": { + "view_model": { "materialized": "table" } } @@ -155,7 +155,7 @@ def test_simple_dependency(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") @@ -163,10 +163,10 @@ def test_simple_dependency(self): # config, table are disabled self.assertFalse('config' in created_models) - self.assertFalse('table' in created_models) + self.assertFalse('table_model' in created_models) - self.assertTrue('view' in created_models) - self.assertEqual(created_models['view'], 'table') + self.assertTrue('view_model' in created_models) + self.assertEqual(created_models['view_model'], 'table') self.assertTrue('incremental' in created_models) self.assertEqual(created_models['incremental'], 'table') From dbcd1245f3dae095fff3e7f40521e5f9e63c496f Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 15 Mar 2018 13:13:51 -0400 Subject: [PATCH 03/61] more model name updates --- .../test_schema_test_graph_selection.py | 12 ++++++------ .../integration/016_macro_tests/macros/my_macros.sql | 2 +- .../models-3/{table.sql => table_model.sql} | 0 .../models-4/{table.sql => table_model.sql} | 0 .../025_duplicate_model_test/test_duplicate_model.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename test/integration/025_duplicate_model_test/models-3/{table.sql => table_model.sql} (100%) rename test/integration/025_duplicate_model_test/models-4/{table.sql => table_model.sql} (100%) diff --git a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py index 8a4e053ad81..d01e490123f 100644 --- a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py @@ -56,7 +56,7 @@ def test__postgres__schema_tests_no_specifiers(self): None, None, ['unique_emails_email', - 'unique_table_id', + 'unique_table_model_id', 'unique_users_id', 'unique_users_rollup_gender'] ) @@ -98,7 +98,7 @@ def test__postgres__schema_tests_specify_exclude_only(self): self.run_schema_and_assert( None, ['users_rollup'], - ['unique_emails_email', 'unique_table_id', 'unique_users_id'] + ['unique_emails_email', 'unique_table_model_id', 'unique_users_id'] ) @attr(type='postgres') @@ -116,7 +116,7 @@ def test__postgres__schema_tests_with_glob(self): self.run_schema_and_assert( ['*'], ['users'], - ['unique_emails_email', 'unique_table_id', 'unique_users_rollup_gender'] + ['unique_emails_email', 'unique_table_model_id', 'unique_users_rollup_gender'] ) @attr(type='postgres') @@ -124,15 +124,15 @@ def test__postgres__schema_tests_dep_package_only(self): self.run_schema_and_assert( ['dbt_integration_project'], None, - ['unique_table_id'] + ['unique_table_model_id'] ) @attr(type='postgres') def test__postgres__schema_tests_model_in_dep_pkg(self): self.run_schema_and_assert( - ['dbt_integration_project.table'], + ['dbt_integration_project.table_model'], None, - ['unique_table_id'] + ['unique_table_model_id'] ) @attr(type='postgres') diff --git a/test/integration/016_macro_tests/macros/my_macros.sql b/test/integration/016_macro_tests/macros/my_macros.sql index 8202b94e295..e929d0d96e2 100644 --- a/test/integration/016_macro_tests/macros/my_macros.sql +++ b/test/integration/016_macro_tests/macros/my_macros.sql @@ -10,6 +10,6 @@ {% macro with_ref() %} - {{ ref('table') }} + {{ ref('table_model') }} {% endmacro %} diff --git a/test/integration/025_duplicate_model_test/models-3/table.sql b/test/integration/025_duplicate_model_test/models-3/table_model.sql similarity index 100% rename from test/integration/025_duplicate_model_test/models-3/table.sql rename to test/integration/025_duplicate_model_test/models-3/table_model.sql diff --git a/test/integration/025_duplicate_model_test/models-4/table.sql b/test/integration/025_duplicate_model_test/models-4/table_model.sql similarity index 100% rename from test/integration/025_duplicate_model_test/models-4/table.sql rename to test/integration/025_duplicate_model_test/models-4/table_model.sql diff --git a/test/integration/025_duplicate_model_test/test_duplicate_model.py b/test/integration/025_duplicate_model_test/test_duplicate_model.py index e9250c5db39..b8280b5ee4f 100644 --- a/test/integration/025_duplicate_model_test/test_duplicate_model.py +++ b/test/integration/025_duplicate_model_test/test_duplicate_model.py @@ -149,7 +149,7 @@ def test_duplicate_model_disabled_across_packages(self): except CompilationException: self.fail( "Compilation Exception raised on disabled model") - query = "select 1 from {schema}.table" \ + query = "select 1 from {schema}.table_model" \ .format(schema=self.unique_schema()) result = self.run_sql(query, fetch="one")[0] assert result == 1 From 5032d1b1f18a6d6b5d721089292435828ae9bcab Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 15 Mar 2018 14:25:39 -0400 Subject: [PATCH 04/61] no more quotes! --- Makefile | 9 ++++----- dbt/adapters/postgres.py | 2 +- .../models/{view.sql => view_model.sql} | 0 .../001_simple_copy_test/test_simple_copy.py | 8 ++++---- .../models/{view.sql => view_model.sql} | 0 .../models-2/{view.sql => view_model.sql} | 0 .../models-3/dependent.sql | 2 +- .../models-3/{view.sql => view_model.sql} | 0 .../models/{view.sql => view_model.sql} | 0 .../test_profile_config.py | 2 +- .../013_context_var_tests/test_context_vars.py | 4 ++-- .../create_view__dbt_tmp.sql | 2 +- .../models/{view.sql => view_model.sql} | 0 .../test_runtime_materialization.py | 18 +++++++++--------- .../update.sql | 2 +- .../021_concurrency_test/models/dep.sql | 2 +- .../models/{view.sql => view_model.sql} | 0 .../021_concurrency_test/test_concurrency.py | 8 ++++---- .../022_bigquery_test/models/schema.yml | 4 ++-- .../models/{view.sql => view_model.sql} | 0 20 files changed, 31 insertions(+), 32 deletions(-) rename test/integration/001_simple_copy_test/models/{view.sql => view_model.sql} (100%) rename test/integration/010_permission_tests/models/{view.sql => view_model.sql} (100%) rename test/integration/011_invalid_model_tests/models-2/{view.sql => view_model.sql} (100%) rename test/integration/011_invalid_model_tests/models-3/{view.sql => view_model.sql} (100%) rename test/integration/012_profile_config_tests/models/{view.sql => view_model.sql} (100%) rename test/integration/017_runtime_materialization_tests/models/{view.sql => view_model.sql} (100%) rename test/integration/021_concurrency_test/models/{view.sql => view_model.sql} (100%) rename test/integration/022_bigquery_test/models/{view.sql => view_model.sql} (100%) diff --git a/Makefile b/Makefile index c6d937a9ba9..441b4fc0d5c 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,7 @@ test-integration: @echo "Integration test run starting..." @time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py36,integration-snowflake-py27,integration-snowflake-py36,integration-bigquery-py27,integration-bigquery-py36 -test-new: - @echo "Test run starting..." - @echo "Changed test files:" - @echo "${changed_tests}" - @docker-compose run test /usr/src/app/test/runner.sh ${changed_tests} + +test-quick: + @echo "Integration test run starting..." + @time docker-compose run test tox -e integration-postgres-py27,unit-py27 -- -x diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres.py index 2cd35cec92a..bb1f3626886 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres.py @@ -224,7 +224,7 @@ def load_csv_rows(cls, profile, schema, table_name, agate_table): ", ".join("%s" for _ in agate_table.column_names))) sql = ('insert into {} ({}) values {}' - .format(cls.get_schema_and_table(schema, table_name), + .format(cls.get_schema_and_table(profile, schema, table_name), cols_sql, ",\n".join(placeholders))) diff --git a/test/integration/001_simple_copy_test/models/view.sql b/test/integration/001_simple_copy_test/models/view_model.sql similarity index 100% rename from test/integration/001_simple_copy_test/models/view.sql rename to test/integration/001_simple_copy_test/models/view_model.sql diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index fcf720f7aac..e11e65ea07a 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -27,7 +27,7 @@ def test__postgres__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -35,7 +35,7 @@ def test__postgres__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -60,7 +60,7 @@ def test__snowflake__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -68,6 +68,6 @@ def test__snowflake__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") diff --git a/test/integration/010_permission_tests/models/view.sql b/test/integration/010_permission_tests/models/view_model.sql similarity index 100% rename from test/integration/010_permission_tests/models/view.sql rename to test/integration/010_permission_tests/models/view_model.sql diff --git a/test/integration/011_invalid_model_tests/models-2/view.sql b/test/integration/011_invalid_model_tests/models-2/view_model.sql similarity index 100% rename from test/integration/011_invalid_model_tests/models-2/view.sql rename to test/integration/011_invalid_model_tests/models-2/view_model.sql diff --git a/test/integration/011_invalid_model_tests/models-3/dependent.sql b/test/integration/011_invalid_model_tests/models-3/dependent.sql index acf363bb859..7e8723a868a 100644 --- a/test/integration/011_invalid_model_tests/models-3/dependent.sql +++ b/test/integration/011_invalid_model_tests/models-3/dependent.sql @@ -1,4 +1,4 @@ -- view is disabled (defined in-model) -select * from {{ ref('view') }} +select * from {{ ref('view_model') }} diff --git a/test/integration/011_invalid_model_tests/models-3/view.sql b/test/integration/011_invalid_model_tests/models-3/view_model.sql similarity index 100% rename from test/integration/011_invalid_model_tests/models-3/view.sql rename to test/integration/011_invalid_model_tests/models-3/view_model.sql diff --git a/test/integration/012_profile_config_tests/models/view.sql b/test/integration/012_profile_config_tests/models/view_model.sql similarity index 100% rename from test/integration/012_profile_config_tests/models/view.sql rename to test/integration/012_profile_config_tests/models/view_model.sql diff --git a/test/integration/012_profile_config_tests/test_profile_config.py b/test/integration/012_profile_config_tests/test_profile_config.py index fbfdaf67df5..c9969020f69 100644 --- a/test/integration/012_profile_config_tests/test_profile_config.py +++ b/test/integration/012_profile_config_tests/test_profile_config.py @@ -42,6 +42,6 @@ def profile_config(self): def test_deprecated_run_target_config(self): self.run_dbt() - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertFalse('run-target' in dbt.deprecations.active_deprecations) diff --git a/test/integration/013_context_var_tests/test_context_vars.py b/test/integration/013_context_var_tests/test_context_vars.py index dc19d3583c8..315da91be77 100644 --- a/test/integration/013_context_var_tests/test_context_vars.py +++ b/test/integration/013_context_var_tests/test_context_vars.py @@ -89,7 +89,7 @@ def test_env_vars_dev(self): self.run_dbt(['run']) ctx = self.get_ctx_vars() - self.assertEqual(ctx['this'], '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual(ctx['this'], '{}.context__dbt_tmp'.format(self.unique_schema())) self.assertEqual(ctx['this.name'], 'context') self.assertEqual(ctx['this.schema'], self.unique_schema()) self.assertEqual(ctx['this.table'], 'context__dbt_tmp') @@ -111,7 +111,7 @@ def test_env_vars_prod(self): self.run_dbt(['run', '--target', 'prod']) ctx = self.get_ctx_vars() - self.assertEqual(ctx['this'], '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual(ctx['this'], '{}.context__dbt_tmp'.format(self.unique_schema())) self.assertEqual(ctx['this.name'], 'context') self.assertEqual(ctx['this.schema'], self.unique_schema()) self.assertEqual(ctx['this.table'], 'context__dbt_tmp') diff --git a/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql b/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql index eeb4d2bca2d..6d92f998671 100644 --- a/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql +++ b/test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql @@ -1,4 +1,4 @@ -create view {schema}.view__dbt_tmp as ( +create view {schema}.view_model__dbt_tmp as ( select 1 as id ); diff --git a/test/integration/017_runtime_materialization_tests/models/view.sql b/test/integration/017_runtime_materialization_tests/models/view_model.sql similarity index 100% rename from test/integration/017_runtime_materialization_tests/models/view.sql rename to test/integration/017_runtime_materialization_tests/models/view_model.sql diff --git a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py index a2acbad9b8e..4cdf83ac2ba 100644 --- a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py +++ b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py @@ -21,7 +21,7 @@ def test_full_refresh(self): # initial full-refresh should have no effect self.run_dbt(['run', '--full-refresh']) - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -34,7 +34,7 @@ def test_full_refresh(self): self.run_dbt(['run', '--full-refresh']) - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -42,7 +42,7 @@ def test_full_refresh(self): def test_non_destructive(self): self.run_dbt(['run', '--non-destructive']) - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") self.assertTableDoesNotExist('dependent_view') @@ -52,7 +52,7 @@ def test_non_destructive(self): self.run_dbt(['run', '--non-destructive']) self.assertTableDoesExist('dependent_view') - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -60,7 +60,7 @@ def test_non_destructive(self): def test_full_refresh_and_non_destructive(self): self.run_dbt(['run', '--full-refresh', '--non-destructive']) - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") self.assertTableDoesNotExist('dependent_view') @@ -71,7 +71,7 @@ def test_full_refresh_and_non_destructive(self): self.run_dbt(['run', '--full-refresh', '--non-destructive']) self.assertTableDoesExist('dependent_view') - self.assertTablesEqual("seed","view") + self.assertTablesEqual("seed","view_model") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -80,7 +80,7 @@ def test_full_refresh_and_non_destructive(self): def test_delete__dbt_tmp_relation(self): # This creates a __dbt_tmp view - make sure it doesn't interfere with the dbt run self.run_sql_file("test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql") - self.run_dbt(['run', '--model', 'view']) + self.run_dbt(['run', '--model', 'view_model']) - self.assertTableDoesNotExist('view__dbt_tmp') - self.assertTablesEqual("seed","view") + self.assertTableDoesNotExist('view__model_dbt_tmp') + self.assertTablesEqual("seed","view_model") diff --git a/test/integration/017_runtime_materialization_tests/update.sql b/test/integration/017_runtime_materialization_tests/update.sql index f5a37926d81..5ded8d56287 100644 --- a/test/integration/017_runtime_materialization_tests/update.sql +++ b/test/integration/017_runtime_materialization_tests/update.sql @@ -6,7 +6,7 @@ create view {schema}.dependent_view as ( select count(*) from {schema}.materialized union all - select count(*) from {schema}.view + select count(*) from {schema}.view_model union all select count(*) from {schema}.incremental diff --git a/test/integration/021_concurrency_test/models/dep.sql b/test/integration/021_concurrency_test/models/dep.sql index 046bab975be..746be81aa64 100644 --- a/test/integration/021_concurrency_test/models/dep.sql +++ b/test/integration/021_concurrency_test/models/dep.sql @@ -4,4 +4,4 @@ ) }} -select * from {{ref('view')}} +select * from {{ref('view_model')}} diff --git a/test/integration/021_concurrency_test/models/view.sql b/test/integration/021_concurrency_test/models/view_model.sql similarity index 100% rename from test/integration/021_concurrency_test/models/view.sql rename to test/integration/021_concurrency_test/models/view_model.sql diff --git a/test/integration/021_concurrency_test/test_concurrency.py b/test/integration/021_concurrency_test/test_concurrency.py index d55db512396..afa4d46a6db 100644 --- a/test/integration/021_concurrency_test/test_concurrency.py +++ b/test/integration/021_concurrency_test/test_concurrency.py @@ -23,7 +23,7 @@ def test__postgres__concurrency(self): self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "view_model") self.assertTablesEqual("seed", "dep") self.assertTablesEqual("seed", "table_a") self.assertTablesEqual("seed", "table_b") @@ -34,7 +34,7 @@ def test__postgres__concurrency(self): self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "view_model") self.assertTablesEqual("seed", "dep") self.assertTablesEqual("seed", "table_a") self.assertTablesEqual("seed", "table_b") @@ -49,7 +49,7 @@ def test__snowflake__concurrency(self): self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "view_model") self.assertTablesEqual("seed", "dep") self.assertTablesEqual("seed", "table_a") self.assertTablesEqual("seed", "table_b") @@ -58,7 +58,7 @@ def test__snowflake__concurrency(self): self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view") + self.assertTablesEqual("seed", "view_model") self.assertTablesEqual("seed", "dep") self.assertTablesEqual("seed", "table_a") self.assertTablesEqual("seed", "table_b") diff --git a/test/integration/022_bigquery_test/models/schema.yml b/test/integration/022_bigquery_test/models/schema.yml index 63aa552c6b9..88114ee08a4 100644 --- a/test/integration/022_bigquery_test/models/schema.yml +++ b/test/integration/022_bigquery_test/models/schema.yml @@ -1,5 +1,5 @@ -view: +view_model: constraints: not_null: - id @@ -10,7 +10,7 @@ view: - dupe # fails was_materialized: - - {name: view, type: view} + - {name: view_model, type: view} table_model: constraints: diff --git a/test/integration/022_bigquery_test/models/view.sql b/test/integration/022_bigquery_test/models/view_model.sql similarity index 100% rename from test/integration/022_bigquery_test/models/view.sql rename to test/integration/022_bigquery_test/models/view_model.sql From 94502bc47e1560e88dac8d1984b3d52a91dc9d76 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 23 Mar 2018 21:16:22 -0400 Subject: [PATCH 05/61] quieter logs --- Makefile | 2 +- dbt/logger.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 441b4fc0d5c..ca6a0dfe085 100644 --- a/Makefile +++ b/Makefile @@ -20,4 +20,4 @@ test-integration: test-quick: @echo "Integration test run starting..." - @time docker-compose run test tox -e integration-postgres-py27,unit-py27 -- -x + @time docker-compose run test tox -e integration-snowflake-py27 -- -x diff --git a/dbt/logger.py b/dbt/logger.py index 301c3a72bc3..2ad662aabe8 100644 --- a/dbt/logger.py +++ b/dbt/logger.py @@ -33,6 +33,14 @@ logger.setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.CRITICAL) +# Quiet these down in the logs +logging.getLogger('botocore').setLevel(logging.INFO) +logging.getLogger('requests').setLevel(logging.INFO) +logging.getLogger('urllib3').setLevel(logging.INFO) +logging.getLogger('google').setLevel(logging.INFO) +logging.getLogger('snowflake.connector').setLevel(logging.INFO) +logging.getLogger('parsedatetime').setLevel(logging.INFO) + # Redirect warnings through our logging setup # They will be logged to a file below logging.captureWarnings(True) From ee3406f0324863c27178bccf72362ce87bfac8e1 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 27 Mar 2018 11:12:53 -0400 Subject: [PATCH 06/61] many changes to fix broken code, tests --- dbt/adapters/snowflake.py | 12 ++-- .../macros/materializations/archive.sql | 52 +++++++++--------- .../models/compound_sort.sql | 2 +- .../001_simple_copy_test/models/disabled.sql | 2 +- .../models/incremental.sql | 2 +- .../models/interleaved_sort.sql | 2 +- .../models/materialized.sql | 2 +- .../models/view_model.sql | 2 +- .../models/incremental.sql | 2 +- .../models/materialized.sql | 2 +- .../002_varchar_widening_test/seed.sql | 4 +- .../002_varchar_widening_test/update.sql | 4 +- .../models/ephemeral_copy.sql | 2 +- .../models/incremental_copy.sql | 2 +- .../models/materialized_copy.sql | 2 +- .../models/view_copy.sql | 2 +- .../003_simple_reference_test/seed.sql | 8 +-- .../test_simple_reference.py | 14 ++--- .../003_simple_reference_test/update.sql | 6 +- .../invalidate_postgres.sql | 4 +- .../invalidate_snowflake.sql | 4 +- .../004_simple_archive_test/seed.sql | 10 ++-- .../test_simple_archive.py | 4 +- .../004_simple_archive_test/update.sql | 10 ++-- .../models/base_users.sql | 2 +- .../007_graph_selection_tests/seed.sql | 8 +-- .../009_data_tests_test/models/table_copy.sql | 2 +- test/integration/009_data_tests_test/seed.sql | 4 +- .../models/view_model.sql | 2 +- .../models-2/view_model.sql | 2 +- .../models-3/view_model.sql | 2 +- .../models/view_model.sql | 2 +- .../models/incremental.sql | 2 +- .../models/materialized.sql | 2 +- .../models/view_model.sql | 2 +- .../models/materialized.sql | 2 +- .../020_ephemeral_test/models/base/base.sql | 2 +- test/integration/020_ephemeral_test/seed.sql | 4 +- .../021_concurrency_test/models/invalid.sql | 2 +- .../021_concurrency_test/models/table_a.sql | 2 +- .../021_concurrency_test/models/table_b.sql | 2 +- .../models/view_model.sql | 2 +- .../integration/021_concurrency_test/seed.sql | 4 +- .../021_concurrency_test/update.sql | 2 +- .../024_custom_schema_test/models/view_1.sql | 2 +- .../024_custom_schema_test/seed.sql | 12 ++-- test/integration/base.py | 55 +++++++++++-------- 47 files changed, 143 insertions(+), 134 deletions(-) diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake.py index 437dd4a2805..748805846b9 100644 --- a/dbt/adapters/snowflake.py +++ b/dbt/adapters/snowflake.py @@ -107,11 +107,13 @@ def query_for_existing(cls, profile, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): schemas = [schemas] - schemas_upper = ["'{}'".format(schema.upper()) for schema in schemas] - schema_list = ",".join(schemas_upper) + schemas = ["upper('{}')".format(schema) for schema in schemas] + schema_list = ",".join(schemas) + # TODO: This lower() is a bad idea... + # Instead, we should wrap table names in a class sql = """ - select table_name as name, table_type as type + select lower(table_name) as name, table_type as type from information_schema.tables where upper(table_schema) in ({schema_list}) """.format(schema_list=schema_list).strip() # noqa @@ -170,8 +172,8 @@ def check_schema_exists(cls, profile, schema, model_name=None): sql = """ select count(*) from information_schema.schemata - where upper(schema_name) = '{schema}' - """.format(schema=schema.upper()).strip() # noqa + where upper(schema_name) = upper('{schema}') + """.format(schema=schema).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name, select_schema=False, diff --git a/dbt/include/global_project/macros/materializations/archive.sql b/dbt/include/global_project/macros/materializations/archive.sql index 68747b0ffe3..658fb054284 100644 --- a/dbt/include/global_project/macros/materializations/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive.sql @@ -14,7 +14,7 @@ ), - "archived_data" as ( + archived_data as ( select {% for col in adapter.get_columns_in_table(source_schema, source_table) %} @@ -28,45 +28,45 @@ ), - "insertions" as ( + insertions as ( select - "current_data".*, + current_data.*, null::timestamp as "valid_to" - from "current_data" - left outer join "archived_data" - on "archived_data"."dbt_pk" = "current_data"."dbt_pk" - where "archived_data"."dbt_pk" is null or ( - "archived_data"."dbt_pk" is not null and - "current_data"."dbt_updated_at" > "archived_data"."dbt_updated_at" and - "archived_data"."tmp_valid_to" is null + from current_data + left outer join archived_data + on archived_data."dbt_pk" = current_data."dbt_pk" + where archived_data."dbt_pk" is null or ( + archived_data."dbt_pk" is not null and + current_data."dbt_updated_at" > archived_data."dbt_updated_at" and + archived_data."tmp_valid_to" is null ) ), - "updates" as ( + updates as ( select - "archived_data".*, - "current_data"."dbt_updated_at" as "valid_to" - from "current_data" - left outer join "archived_data" - on "archived_data"."dbt_pk" = "current_data"."dbt_pk" - where "archived_data"."dbt_pk" is not null - and "archived_data"."dbt_updated_at" < "current_data"."dbt_updated_at" - and "archived_data"."tmp_valid_to" is null + archived_data.*, + current_data."dbt_updated_at" as "valid_to" + from current_data + left outer join archived_data + on archived_data."dbt_pk" = current_data."dbt_pk" + where archived_data."dbt_pk" is not null + and archived_data."dbt_updated_at" < current_data."dbt_updated_at" + and archived_data."tmp_valid_to" is null ), - "merged" as ( + merged as ( - select *, 'update' as "change_type" from "updates" + select *, 'update' as "change_type" from updates union all - select *, 'insert' as "change_type" from "insertions" + select *, 'insert' as "change_type" from insertions ) select *, md5("dbt_pk" || '|' || "dbt_updated_at") as "scd_id" - from "merged" + from merged {% endmacro %} @@ -130,9 +130,9 @@ to_table=target_table) }} {% call statement('main') -%} - update {{ target_schema }}.{{ identifier }} set "valid_to" = "tmp"."valid_to" - from {{ tmp_identifier }} as "tmp" - where "tmp"."scd_id" = {{ target_schema }}.{{ identifier }}."scd_id" + update {{ target_schema }}.{{ identifier }} set "valid_to" = tmp."valid_to" + from {{ tmp_identifier }} as tmp + where tmp."scd_id" = {{ target_schema }}.{{ identifier }}."scd_id" and "change_type" = 'update'; insert into {{ target_schema }}.{{ identifier }} ( diff --git a/test/integration/001_simple_copy_test/models/compound_sort.sql b/test/integration/001_simple_copy_test/models/compound_sort.sql index 236de4a08b9..9ccc9374e3a 100644 --- a/test/integration/001_simple_copy_test/models/compound_sort.sql +++ b/test/integration/001_simple_copy_test/models/compound_sort.sql @@ -6,4 +6,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/001_simple_copy_test/models/disabled.sql b/test/integration/001_simple_copy_test/models/disabled.sql index 44d93ff8b83..0f9d1d12c1c 100644 --- a/test/integration/001_simple_copy_test/models/disabled.sql +++ b/test/integration/001_simple_copy_test/models/disabled.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/001_simple_copy_test/models/incremental.sql b/test/integration/001_simple_copy_test/models/incremental.sql index 591b77faa5d..ec68f6c4824 100644 --- a/test/integration/001_simple_copy_test/models/incremental.sql +++ b/test/integration/001_simple_copy_test/models/incremental.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/001_simple_copy_test/models/interleaved_sort.sql b/test/integration/001_simple_copy_test/models/interleaved_sort.sql index ff4eba3c19b..02e8c48812c 100644 --- a/test/integration/001_simple_copy_test/models/interleaved_sort.sql +++ b/test/integration/001_simple_copy_test/models/interleaved_sort.sql @@ -6,4 +6,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/001_simple_copy_test/models/materialized.sql b/test/integration/001_simple_copy_test/models/materialized.sql index 5130aa9e4db..91387ccacda 100644 --- a/test/integration/001_simple_copy_test/models/materialized.sql +++ b/test/integration/001_simple_copy_test/models/materialized.sql @@ -5,4 +5,4 @@ }} -- this is a unicode character: å -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/001_simple_copy_test/models/view_model.sql b/test/integration/001_simple_copy_test/models/view_model.sql index 014818a472a..c7a2b6a6996 100644 --- a/test/integration/001_simple_copy_test/models/view_model.sql +++ b/test/integration/001_simple_copy_test/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/002_varchar_widening_test/models/incremental.sql b/test/integration/002_varchar_widening_test/models/incremental.sql index 82992c64e1a..d7e63d2a5d1 100644 --- a/test/integration/002_varchar_widening_test/models/incremental.sql +++ b/test/integration/002_varchar_widening_test/models/incremental.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/002_varchar_widening_test/models/materialized.sql b/test/integration/002_varchar_widening_test/models/materialized.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/002_varchar_widening_test/models/materialized.sql +++ b/test/integration/002_varchar_widening_test/models/materialized.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/002_varchar_widening_test/seed.sql b/test/integration/002_varchar_widening_test/seed.sql index 43d320f45a6..7406b18a067 100644 --- a/test/integration/002_varchar_widening_test/seed.sql +++ b/test/integration/002_varchar_widening_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -7,7 +7,7 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/002_varchar_widening_test/update.sql b/test/integration/002_varchar_widening_test/update.sql index e216a3ac798..7398d436534 100644 --- a/test/integration/002_varchar_widening_test/update.sql +++ b/test/integration/002_varchar_widening_test/update.sql @@ -1,6 +1,6 @@ -ALTER TABLE "{schema}"."seed" ALTER COLUMN gender TYPE varchar(300); +ALTER TABLE {schema}.seed ALTER COLUMN gender TYPE varchar(300); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Annie', 'Reynolds', 'areynolds0@nifty.com', 'Amerisource Bergen', '133.30.242.211'), ('Doris', 'Wood', 'dwood1@skyrock.com', 'Bliss World, LLC', '128.229.89.207'), ('Andrea', 'Ray', 'aray2@google.co.jp', 'Nelco Laboratories, Inc.', '109.74.153.45'), diff --git a/test/integration/003_simple_reference_test/models/ephemeral_copy.sql b/test/integration/003_simple_reference_test/models/ephemeral_copy.sql index 66823f4030a..c6c2c53e900 100644 --- a/test/integration/003_simple_reference_test/models/ephemeral_copy.sql +++ b/test/integration/003_simple_reference_test/models/ephemeral_copy.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/models/incremental_copy.sql b/test/integration/003_simple_reference_test/models/incremental_copy.sql index 82992c64e1a..d7e63d2a5d1 100644 --- a/test/integration/003_simple_reference_test/models/incremental_copy.sql +++ b/test/integration/003_simple_reference_test/models/incremental_copy.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/models/materialized_copy.sql b/test/integration/003_simple_reference_test/models/materialized_copy.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/003_simple_reference_test/models/materialized_copy.sql +++ b/test/integration/003_simple_reference_test/models/materialized_copy.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/models/view_copy.sql b/test/integration/003_simple_reference_test/models/view_copy.sql index 15f6caffe13..e0560b694c9 100644 --- a/test/integration/003_simple_reference_test/models/view_copy.sql +++ b/test/integration/003_simple_reference_test/models/view_copy.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/003_simple_reference_test/seed.sql b/test/integration/003_simple_reference_test/seed.sql index 550964ea8a2..63f5384a442 100644 --- a/test/integration/003_simple_reference_test/seed.sql +++ b/test/integration/003_simple_reference_test/seed.sql @@ -1,13 +1,13 @@ -create table "{schema}"."summary_expected" ( +create table {schema}.summary_expected ( gender VARCHAR(10), ct BIGINT ); -insert into "{schema}"."summary_expected" (gender, ct) values +insert into {schema}.summary_expected (gender, ct) values ('Female', 40), ('Male', 60); -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -16,7 +16,7 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/003_simple_reference_test/test_simple_reference.py b/test/integration/003_simple_reference_test/test_simple_reference.py index 91b9336b12d..bdbef08807f 100644 --- a/test/integration/003_simple_reference_test/test_simple_reference.py +++ b/test/integration/003_simple_reference_test/test_simple_reference.py @@ -148,7 +148,7 @@ def test__snowflake__simple_reference_with_models(self): self.assertTablesEqual("seed","materialized_copy") created_models = self.get_models_in_schema() - self.assertTrue('materialized_copy' in created_models) + self.assertTrue('MATERIALIZED_COPY' in created_models) @attr(type='snowflake') def test__snowflake__simple_reference_with_models_and_children(self): @@ -178,10 +178,10 @@ def test__snowflake__simple_reference_with_models_and_children(self): # make sure this wasn't errantly materialized self.assertFalse('ephemeral_copy' in created_models) - self.assertTrue('materialized_copy' in created_models) - self.assertTrue('materialized_summary' in created_models) - self.assertEqual(created_models['materialized_copy'], 'table') - self.assertEqual(created_models['materialized_summary'], 'table') + self.assertTrue('MATERIALIZED_COPY' in created_models) + self.assertTrue('MATERIALIZED_SUMMARY' in created_models) + self.assertEqual(created_models['MATERIALIZED_COPY'], 'table') + self.assertEqual(created_models['MATERIALIZED_SUMMARY'], 'table') - self.assertTrue('ephemeral_summary' in created_models) - self.assertEqual(created_models['ephemeral_summary'], 'table') + self.assertTrue('EPHEMERAL_SUMMARY' in created_models) + self.assertEqual(created_models['EPHEMERAL_SUMMARY'], 'table') diff --git a/test/integration/003_simple_reference_test/update.sql b/test/integration/003_simple_reference_test/update.sql index 816e4876078..abf3c0b340a 100644 --- a/test/integration/003_simple_reference_test/update.sql +++ b/test/integration/003_simple_reference_test/update.sql @@ -1,10 +1,10 @@ -truncate table "{schema}"."summary_expected"; -insert into "{schema}"."summary_expected" (gender, ct) values +truncate table {schema}.summary_expected; +insert into {schema}.summary_expected (gender, ct) values ('Female', 94), ('Male', 106); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), ('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), ('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), diff --git a/test/integration/004_simple_archive_test/invalidate_postgres.sql b/test/integration/004_simple_archive_test/invalidate_postgres.sql index 779f9b09804..30902625059 100644 --- a/test/integration/004_simple_archive_test/invalidate_postgres.sql +++ b/test/integration/004_simple_archive_test/invalidate_postgres.sql @@ -1,12 +1,12 @@ -- update records 11 - 21. Change email and updated_at field -update "{schema}"."seed" set +update {schema}.seed set "updated_at" = "updated_at" + interval '1 hour', "email" = 'new_' || "email" where "id" >= 10 and "id" <= 20; -- invalidate records 11 - 21 -update "{schema}"."archive_expected" set +update {schema}.archive_expected set "valid_to" = "updated_at" + interval '1 hour' where "id" >= 10 and "id" <= 20; diff --git a/test/integration/004_simple_archive_test/invalidate_snowflake.sql b/test/integration/004_simple_archive_test/invalidate_snowflake.sql index 276c3da6a4d..879d2966d05 100644 --- a/test/integration/004_simple_archive_test/invalidate_snowflake.sql +++ b/test/integration/004_simple_archive_test/invalidate_snowflake.sql @@ -1,12 +1,12 @@ -- update records 11 - 21. Change email and updated_at field -update "{schema}"."seed" set +update {schema}.seed set "updated_at" = DATEADD(hour, 1, "updated_at"), "email" = 'new_' || "email" where "id" >= 10 and "id" <= 20; -- invalidate records 11 - 21 -update "{schema}"."archive_expected" set +update {schema}.archive_expected set "valid_to" = DATEADD(hour, 1, "updated_at") where "id" >= 10 and "id" <= 20; diff --git a/test/integration/004_simple_archive_test/seed.sql b/test/integration/004_simple_archive_test/seed.sql index c3256b7d735..99b5bf4bbe5 100644 --- a/test/integration/004_simple_archive_test/seed.sql +++ b/test/integration/004_simple_archive_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( "id" INTEGER, "first_name" VARCHAR(50), "last_name" VARCHAR(50), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( "updated_at" TIMESTAMP WITHOUT TIME ZONE ); -create table "{schema}"."archive_expected" ( +create table {schema}.archive_expected ( "id" INTEGER, "first_name" VARCHAR(50), "last_name" VARCHAR(50), @@ -26,7 +26,7 @@ create table "{schema}"."archive_expected" ( -- seed inserts -insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +insert into {schema}.seed ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values (1, 'Judith', 'Kennedy', 'jkennedy0@phpbb.com', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', 'akelly1@eepurl.com', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), @@ -50,7 +50,7 @@ insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender -- populate archive table -insert into "{schema}"."archive_expected" ( +insert into {schema}.archive_expected ( "id", "first_name", "last_name", @@ -77,4 +77,4 @@ select null::timestamp as valid_to, "updated_at" as dbt_updated_at, md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as scd_id -from "{schema}"."seed"; +from {schema}.seed; diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py index 58bb9f4cc51..51b7bce4d74 100644 --- a/test/integration/004_simple_archive_test/test_simple_archive.py +++ b/test/integration/004_simple_archive_test/test_simple_archive.py @@ -58,11 +58,11 @@ def test__snowflake__simple_archive(self): self.run_dbt(["archive"]) - self.assertTablesEqual("archive_expected","archive_actual") + self.assertTablesEqual("ARCHIVE_EXPECTED","ARCHIVE_ACTUAL") self.run_sql_file("test/integration/004_simple_archive_test/invalidate_snowflake.sql") self.run_sql_file("test/integration/004_simple_archive_test/update.sql") self.run_dbt(["archive"]) - self.assertTablesEqual("archive_expected","archive_actual") + self.assertTablesEqual("ARCHIVE_EXPECTED","ARCHIVE_ACTUAL") diff --git a/test/integration/004_simple_archive_test/update.sql b/test/integration/004_simple_archive_test/update.sql index d574e254702..32b954d7f0c 100644 --- a/test/integration/004_simple_archive_test/update.sql +++ b/test/integration/004_simple_archive_test/update.sql @@ -1,6 +1,6 @@ -- insert v2 of the 11 - 21 records -insert into "{schema}"."archive_expected" ( +insert into {schema}.archive_expected ( "id", "first_name", "last_name", @@ -27,12 +27,12 @@ select null::timestamp as "valid_to", "updated_at" as "dbt_updated_at", md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "scd_id" -from "{schema}"."seed" +from {schema}.seed where "id" >= 10 and "id" <= 20; -- insert 10 new records -insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values +insert into {schema}.seed ("id", "first_name", "last_name", "email", "gender", "ip_address", "updated_at") values (21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), (22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), (23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), @@ -46,7 +46,7 @@ insert into "{schema}"."seed" ("id", "first_name", "last_name", "email", "gender -- add these new records to the archive table -insert into "{schema}"."archive_expected" ( +insert into {schema}.archive_expected ( "id", "first_name", "last_name", @@ -73,5 +73,5 @@ select null::timestamp as "valid_to", "updated_at" as "dbt_updated_at", md5("id" || '-' || "first_name" || '|' || "updated_at"::text) as "scd_id" -from "{schema}"."seed" +from {schema}.seed where "id" > 20; diff --git a/test/integration/007_graph_selection_tests/models/base_users.sql b/test/integration/007_graph_selection_tests/models/base_users.sql index dd664b5cf32..fbab29807f3 100644 --- a/test/integration/007_graph_selection_tests/models/base_users.sql +++ b/test/integration/007_graph_selection_tests/models/base_users.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/007_graph_selection_tests/seed.sql b/test/integration/007_graph_selection_tests/seed.sql index 550964ea8a2..63f5384a442 100644 --- a/test/integration/007_graph_selection_tests/seed.sql +++ b/test/integration/007_graph_selection_tests/seed.sql @@ -1,13 +1,13 @@ -create table "{schema}"."summary_expected" ( +create table {schema}.summary_expected ( gender VARCHAR(10), ct BIGINT ); -insert into "{schema}"."summary_expected" (gender, ct) values +insert into {schema}.summary_expected (gender, ct) values ('Female', 40), ('Male', 60); -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -16,7 +16,7 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/009_data_tests_test/models/table_copy.sql b/test/integration/009_data_tests_test/models/table_copy.sql index b7d5d7b745e..56e90a6d93c 100644 --- a/test/integration/009_data_tests_test/models/table_copy.sql +++ b/test/integration/009_data_tests_test/models/table_copy.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/009_data_tests_test/seed.sql b/test/integration/009_data_tests_test/seed.sql index caac3985e6b..445c087cd66 100644 --- a/test/integration/009_data_tests_test/seed.sql +++ b/test/integration/009_data_tests_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( favorite_color VARCHAR(10), id INTEGER, first_name VARCHAR(11), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( ); -INSERT INTO "{schema}"."seed" +INSERT INTO {schema}.seed (favorite_color, id, first_name, email, ip_address, updated_at) VALUES ('blue', 1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), diff --git a/test/integration/010_permission_tests/models/view_model.sql b/test/integration/010_permission_tests/models/view_model.sql index ce40d77a0c0..31dc91a8d51 100644 --- a/test/integration/010_permission_tests/models/view_model.sql +++ b/test/integration/010_permission_tests/models/view_model.sql @@ -1,2 +1,2 @@ -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/011_invalid_model_tests/models-2/view_model.sql b/test/integration/011_invalid_model_tests/models-2/view_model.sql index ae0485cbcf1..162396e7850 100644 --- a/test/integration/011_invalid_model_tests/models-2/view_model.sql +++ b/test/integration/011_invalid_model_tests/models-2/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/011_invalid_model_tests/models-3/view_model.sql b/test/integration/011_invalid_model_tests/models-3/view_model.sql index ae8f688759a..b615ec282d8 100644 --- a/test/integration/011_invalid_model_tests/models-3/view_model.sql +++ b/test/integration/011_invalid_model_tests/models-3/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/012_profile_config_tests/models/view_model.sql b/test/integration/012_profile_config_tests/models/view_model.sql index 15f6caffe13..e0560b694c9 100644 --- a/test/integration/012_profile_config_tests/models/view_model.sql +++ b/test/integration/012_profile_config_tests/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/models/incremental.sql b/test/integration/017_runtime_materialization_tests/models/incremental.sql index 82992c64e1a..d7e63d2a5d1 100644 --- a/test/integration/017_runtime_materialization_tests/models/incremental.sql +++ b/test/integration/017_runtime_materialization_tests/models/incremental.sql @@ -5,4 +5,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/models/materialized.sql b/test/integration/017_runtime_materialization_tests/models/materialized.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/017_runtime_materialization_tests/models/materialized.sql +++ b/test/integration/017_runtime_materialization_tests/models/materialized.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/models/view_model.sql b/test/integration/017_runtime_materialization_tests/models/view_model.sql index 15f6caffe13..e0560b694c9 100644 --- a/test/integration/017_runtime_materialization_tests/models/view_model.sql +++ b/test/integration/017_runtime_materialization_tests/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/018_adapter_ddl_tests/models/materialized.sql b/test/integration/018_adapter_ddl_tests/models/materialized.sql index a950c6bdf09..edd9c8e04bf 100644 --- a/test/integration/018_adapter_ddl_tests/models/materialized.sql +++ b/test/integration/018_adapter_ddl_tests/models/materialized.sql @@ -6,4 +6,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/020_ephemeral_test/models/base/base.sql b/test/integration/020_ephemeral_test/models/base/base.sql index edd3e7c6f46..5d9b6de149b 100644 --- a/test/integration/020_ephemeral_test/models/base/base.sql +++ b/test/integration/020_ephemeral_test/models/base/base.sql @@ -1,3 +1,3 @@ {{ config(materialized='ephemeral') }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/020_ephemeral_test/seed.sql b/test/integration/020_ephemeral_test/seed.sql index 28309b400f8..3c618ccab3a 100644 --- a/test/integration/020_ephemeral_test/seed.sql +++ b/test/integration/020_ephemeral_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/021_concurrency_test/models/invalid.sql b/test/integration/021_concurrency_test/models/invalid.sql index 4ea041f6c75..fa7046f9ebc 100644 --- a/test/integration/021_concurrency_test/models/invalid.sql +++ b/test/integration/021_concurrency_test/models/invalid.sql @@ -4,4 +4,4 @@ ) }} -select a_field_that_does_not_exist from "{{ this.schema }}"."seed" +select a_field_that_does_not_exist from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/models/table_a.sql b/test/integration/021_concurrency_test/models/table_a.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/021_concurrency_test/models/table_a.sql +++ b/test/integration/021_concurrency_test/models/table_a.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/models/table_b.sql b/test/integration/021_concurrency_test/models/table_b.sql index 314afafac59..79dd477a36e 100644 --- a/test/integration/021_concurrency_test/models/table_b.sql +++ b/test/integration/021_concurrency_test/models/table_b.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/models/view_model.sql b/test/integration/021_concurrency_test/models/view_model.sql index 15f6caffe13..e0560b694c9 100644 --- a/test/integration/021_concurrency_test/models/view_model.sql +++ b/test/integration/021_concurrency_test/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from "{{ this.schema }}"."seed" +select * from {{ this.schema }}.seed diff --git a/test/integration/021_concurrency_test/seed.sql b/test/integration/021_concurrency_test/seed.sql index 28309b400f8..3c618ccab3a 100644 --- a/test/integration/021_concurrency_test/seed.sql +++ b/test/integration/021_concurrency_test/seed.sql @@ -1,4 +1,4 @@ -create table "{schema}"."seed" ( +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -8,7 +8,7 @@ create table "{schema}"."seed" ( ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), diff --git a/test/integration/021_concurrency_test/update.sql b/test/integration/021_concurrency_test/update.sql index e78d5d7d6df..8a8d2ac1a8b 100644 --- a/test/integration/021_concurrency_test/update.sql +++ b/test/integration/021_concurrency_test/update.sql @@ -1,4 +1,4 @@ -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), ('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), ('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), diff --git a/test/integration/024_custom_schema_test/models/view_1.sql b/test/integration/024_custom_schema_test/models/view_1.sql index a42ec4a2916..501c773e8f0 100644 --- a/test/integration/024_custom_schema_test/models/view_1.sql +++ b/test/integration/024_custom_schema_test/models/view_1.sql @@ -1,3 +1,3 @@ -select * from "{{ target.schema }}"."seed" +select * from {{ target.schema }}.seed diff --git a/test/integration/024_custom_schema_test/seed.sql b/test/integration/024_custom_schema_test/seed.sql index b041208db00..607759b0781 100644 --- a/test/integration/024_custom_schema_test/seed.sql +++ b/test/integration/024_custom_schema_test/seed.sql @@ -1,6 +1,6 @@ -drop table if exists "{schema}"."seed" cascade; -create table "{schema}"."seed" ( +drop table if exists {schema}.seed cascade; +create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), @@ -9,17 +9,17 @@ create table "{schema}"."seed" ( ip_address VARCHAR(20) ); -drop table if exists "{schema}"."agg" cascade; -create table "{schema}"."agg" ( +drop table if exists {schema}.agg cascade; +create table {schema}.agg ( last_name VARCHAR(50), count BIGINT ); -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values +insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'); -insert into "{schema}"."agg" (last_name, count) values +insert into {schema}.agg (last_name, count) values ('Hunter', 2), ('Walker', 2), ('Ryan', 2); diff --git a/test/integration/base.py b/test/integration/base.py index c7066f4c62d..b5af4249d13 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -184,8 +184,8 @@ def setUp(self): adapter.drop_schema(profile, schema_name, '__test') adapter.create_schema(profile, schema_name, '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.unique_schema())) - self.run_sql('CREATE SCHEMA "{}"'.format(self.unique_schema())) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) + self.run_sql('CREATE SCHEMA {}'.format(self.unique_schema())) def use_default_project(self, overrides=None): # create a dbt_project.yml @@ -232,8 +232,8 @@ def use_profile(self, adapter_type): adapter.drop_schema(profile, self.unique_schema(), '__test') adapter.create_schema(profile, self.unique_schema(), '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.unique_schema())) - self.run_sql('CREATE SCHEMA "{}"'.format(self.unique_schema())) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) + self.run_sql('CREATE SCHEMA {}'.format(self.unique_schema())) def tearDown(self): os.remove(DBT_PROFILES) @@ -250,7 +250,7 @@ def tearDown(self): adapter = get_adapter(self.profile) adapter.drop_schema(self.profile, self.unique_schema(), '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.unique_schema())) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) self.handle.close() @@ -328,8 +328,8 @@ def get_table_columns(self, table, schema=None): sql = """ select column_name, data_type, character_maximum_length from information_schema.columns - where table_name = '{}' - and table_schema = '{}' + where table_name ilike '{}' + and table_schema ilike '{}' order by column_name asc""" result = self.run_sql(sql.format(table, schema), fetch='all') @@ -345,7 +345,7 @@ def get_models_in_schema(self, schema=None): else table_type end as materialization from information_schema.tables - where table_schema = '{}' + where table_schema ilike '{}' order by table_name """ @@ -353,25 +353,21 @@ def get_models_in_schema(self, schema=None): return {model_name: materialization for (model_name, materialization) in result} - def assertTablesEqual(self, table_a, table_b, table_a_schema=None, table_b_schema=None): - table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema - table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema - - self.assertTableColumnsEqual(table_a, table_b, table_a_schema, table_b_schema) - self.assertTableRowCountsEqual(table_a, table_b, table_a_schema, table_b_schema) - + def _assertTablesEqualSql(self, table_a_schema, table_a, table_b_schema, table_b): columns = self.get_table_columns(table_a, table_a_schema) - columns_csv = ", ".join(['"{}"'.format(record[0]) - for record in columns]) - table_sql = "SELECT {} FROM {}" + + if self.adapter_type == 'snowflake': + columns_csv = ", ".join(['"{}"'.format(record[0]) for record in columns]) + else: + columns_csv = ", ".join(['{}'.format(record[0]) for record in columns]) sql = """ SELECT COUNT(*) FROM ( - (SELECT {columns} FROM "{table_a_schema}"."{table_a}" EXCEPT - SELECT {columns} FROM "{table_b_schema}"."{table_b}") + (SELECT {columns} FROM {table_a_schema}.{table_a} EXCEPT + SELECT {columns} FROM {table_b_schema}.{table_b}) UNION ALL - (SELECT {columns} FROM "{table_b_schema}"."{table_b}" EXCEPT - SELECT {columns} FROM "{table_a_schema}"."{table_a}") + (SELECT {columns} FROM {table_b_schema}.{table_b} EXCEPT + SELECT {columns} FROM {table_a_schema}.{table_a}) ) AS a""".format( columns=columns_csv, table_a_schema=table_a_schema, @@ -380,6 +376,17 @@ def assertTablesEqual(self, table_a, table_b, table_a_schema=None, table_b_schem table_b=table_b ) + return sql + + def assertTablesEqual(self, table_a, table_b, table_a_schema=None, table_b_schema=None): + table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema + table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema + + self.assertTableColumnsEqual(table_a, table_b, table_a_schema, table_b_schema) + self.assertTableRowCountsEqual(table_a, table_b, table_a_schema, table_b_schema) + + + sql = self._assertTablesEqualSql(table_a_schema, table_a, table_b_schema, table_b) result = self.run_sql(sql, fetch='one') self.assertEquals( @@ -392,8 +399,8 @@ def assertTableRowCountsEqual(self, table_a, table_b, table_a_schema=None, table table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema - table_a_result = self.run_sql('SELECT COUNT(*) FROM "{}"."{}"'.format(table_a_schema, table_a), fetch='one') - table_b_result = self.run_sql('SELECT COUNT(*) FROM "{}"."{}"'.format(table_b_schema, table_b), fetch='one') + table_a_result = self.run_sql('SELECT COUNT(*) FROM {}.{}'.format(table_a_schema, table_a), fetch='one') + table_b_result = self.run_sql('SELECT COUNT(*) FROM {}.{}'.format(table_b_schema, table_b), fetch='one') self.assertEquals( table_a_result[0], From b27a374c3cb0d36b404ca7f60b6fe6d5332515c9 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 27 Mar 2018 18:22:38 -0400 Subject: [PATCH 07/61] fix for archival --- dbt/adapters/default.py | 13 +++++++++---- dbt/adapters/snowflake.py | 11 ++++++++--- .../global_project/macros/adapters/common.sql | 10 ++++++++++ .../global_project/macros/adapters/snowflake.sql | 11 +++++++++++ .../macros/materializations/bigquery.sql | 4 ++-- .../macros/materializations/helpers.sql | 2 +- .../macros/materializations/incremental.sql | 5 +++-- .../macros/materializations/table.sql | 2 +- .../global_project/macros/materializations/view.sql | 2 +- 9 files changed, 46 insertions(+), 14 deletions(-) diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index 33846a097a7..67909115910 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -610,10 +610,14 @@ def drop_schema(cls, profile, schema, model_name=None): return cls.add_query(profile, sql, model_name) @classmethod - def table_exists(cls, profile, schema, table, model_name=None): + def table_existing_type(cls, profile, schema, table, model_name=None): tables = cls.query_for_existing(profile, schema, model_name) - exists = tables.get(table) is not None - return exists + return tables.get(table) + + @classmethod + def table_exists(cls, profile, schema, table, model_name=None): + rel_type = cls.table_existing_type(profile, schema, table, model_name) + return rel_type is not None @classmethod def already_exists(cls, profile, schema, table, model_name=None): @@ -644,7 +648,8 @@ def get_schema_and_table(cls, profile, schema, table, quote=False, def handle_csv_table(cls, profile, schema, table_name, agate_table, full_refresh=False): existing = cls.query_for_existing(profile, schema) - existing_type = existing.get(table_name) + upcased_existing = {k.upper(): v for k, v in existing.items()} + existing_type = upcased_existing.get(table_name.upper()) if existing_type and existing_type != "table": raise dbt.exceptions.RuntimeException( "Cannot seed to '{}', it is a view".format(table_name)) diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake.py index 748805846b9..a1d276f062b 100644 --- a/dbt/adapters/snowflake.py +++ b/dbt/adapters/snowflake.py @@ -102,6 +102,13 @@ def open_connection(cls, connection): return result + @classmethod + def table_existing_type(cls, profile, schema, table, model_name=None): + # this is a case-insensitive lookup + relations = cls.query_for_existing(profile, schema, model_name) + upcased_relations = {k.upper(): v for (k, v) in relations.items()} + return upcased_relations.get(table.upper()) + @classmethod def query_for_existing(cls, profile, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): @@ -110,10 +117,8 @@ def query_for_existing(cls, profile, schemas, model_name=None): schemas = ["upper('{}')".format(schema) for schema in schemas] schema_list = ",".join(schemas) - # TODO: This lower() is a bad idea... - # Instead, we should wrap table names in a class sql = """ - select lower(table_name) as name, table_type as type + select table_name as name, table_type as type from information_schema.tables where upper(table_schema) in ({schema_list}) """.format(schema_list=schema_list).strip() # noqa diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index b8138b3e62c..39cd75cf8d0 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -63,3 +63,13 @@ {{ column_list_for_create_table(columns) }} ); {% endmacro %} + + +{% macro get_existing_relation_type(existing, identifier) -%} + {{ return(adapter_macro('get_existing_relation_type', existing, identifier)) }} +{%- endmacro %} + +{% macro default__get_existing_relation_type(existing, identifier) -%} + {%- set existing_type = existing.get(identifier) -%} + {{ return(existing_type) }} +{%- endmacro %} diff --git a/dbt/include/global_project/macros/adapters/snowflake.sql b/dbt/include/global_project/macros/adapters/snowflake.sql index 716f67a07dc..4040597008d 100644 --- a/dbt/include/global_project/macros/adapters/snowflake.sql +++ b/dbt/include/global_project/macros/adapters/snowflake.sql @@ -8,3 +8,14 @@ {{ sql }} ); {% endmacro %} + + +{% macro snowflake__get_existing_relation_type(existing, identifier) -%} + {%- set upcased_existing = {} -%} + {%- for k,v in existing.items() -%} + {%- set _ = upcased_existing.update({k.upper(): v}) -%} + {%- endfor -%} + + {%- set existing_type = upcased_existing.get(identifier.upper()) -%} + {{ return(existing_type) }} +{%- endmacro %} diff --git a/dbt/include/global_project/macros/materializations/bigquery.sql b/dbt/include/global_project/macros/materializations/bigquery.sql index f616412a593..7762d2b1863 100644 --- a/dbt/include/global_project/macros/materializations/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/bigquery.sql @@ -3,7 +3,7 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_type = get_existing_relation_type(existing, identifier) -%} {%- if existing_type is not none -%} {%- if existing_type == 'table' and not flags.FULL_REFRESH -%} @@ -58,7 +58,7 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_type = get_existing_relation_type(existing, identifier) -%} {%- set verbose = config.get('verbose', False) -%} {%- set partitions = config.get('partitions') -%} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 1a20316c2f3..3eb190e3e08 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -47,7 +47,7 @@ {% macro drop_if_exists(existing, schema, name) %} - {% set existing_type = existing.get(name) %} + {% set existing_type = get_existing_relation_type(existing, name) %} {% if existing_type is not none %} {{ adapter.drop(schema, name, existing_type) }} {% endif %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index f82f2112fc4..b3260a6ea84 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -22,7 +22,7 @@ {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_type = get_existing_relation_type(existing, identifier) -%} {%- set exists_as_table = (existing_type == 'table') -%} {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} @@ -44,7 +44,8 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model - {% if force_create or not adapter.already_exists(schema, identifier) -%} + {# TODO : Can we use `exists_as_table` here? Or did we need adapter.already_exists() #} + {% if force_create or not exists_as_table -%} {%- call statement('main') -%} {{ create_table_as(False, identifier, sql) }} {%- endcall -%} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 0dfecce3280..c646bb68c3a 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -3,7 +3,7 @@ {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_type = get_existing_relation_type(existing, identifier) -%} {{ drop_if_exists(existing, schema, tmp_identifier) }} diff --git a/dbt/include/global_project/macros/materializations/view.sql b/dbt/include/global_project/macros/materializations/view.sql index b5bba061142..6e172cebd9a 100644 --- a/dbt/include/global_project/macros/materializations/view.sql +++ b/dbt/include/global_project/macros/materializations/view.sql @@ -4,7 +4,7 @@ {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_type = get_existing_relation_type(existing, identifier) -%} {%- set has_transactional_hooks = (hooks | selectattr('transaction', 'equalto', True) | list | length) > 0 %} {%- set should_ignore = non_destructive_mode and existing_type == 'view' %} From d52058f9a143799c571b22f0e219be610a8b677a Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 28 Mar 2018 09:20:12 -0400 Subject: [PATCH 08/61] fixes for bq --- dbt/adapters/bigquery.py | 10 +++++++--- dbt/adapters/default.py | 16 +++++----------- dbt/adapters/postgres.py | 4 ++-- dbt/utils.py | 15 +++++++++++++-- .../022_bigquery_test/models/table_model.sql | 2 +- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/dbt/adapters/bigquery.py b/dbt/adapters/bigquery.py index b8a2827703f..6ec3951c01b 100644 --- a/dbt/adapters/bigquery.py +++ b/dbt/adapters/bigquery.py @@ -429,12 +429,16 @@ def quote(cls, identifier): @classmethod def quote_schema_and_table(cls, profile, schema, table, model_name=None): + return cls.render_relation(profile, + cls.quote(schema), + cls.quote(table)) + + @classmethod + def render_relation(cls, profile, schema, table): connection = cls.get_connection(profile) credentials = connection.get('credentials', {}) project = credentials.get('project') - return '{}.{}.{}'.format(cls.quote(project), - cls.quote(schema), - cls.quote(table)) + return '{}.{}.{}'.format(cls.quote(project), schema, table) @classmethod def convert_text_type(cls, agate_table, col_idx): diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index 67909115910..d300b00a7ee 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -34,7 +34,6 @@ class DefaultAdapter(object): "add_query", "expand_target_column_types", "quote_schema_and_table", - "get_schema_and_table", "execute" ] @@ -140,7 +139,7 @@ def drop(cls, profile, schema, relation, relation_type, model_name=None): @classmethod def drop_relation(cls, profile, schema, rel_name, rel_type, model_name): - relation = cls.get_schema_and_table(profile, schema, rel_name) + relation = cls.render_relation(profile, schema, rel_name) sql = 'drop {} if exists {} cascade'.format(rel_type, relation) connection, cursor = cls.add_query(profile, sql, model_name) @@ -155,14 +154,14 @@ def drop_table(cls, profile, schema, table, model_name): @classmethod def truncate(cls, profile, schema, table, model_name=None): - relation = cls.get_schema_and_table(profile, schema, table) + relation = cls.render_relation(profile, schema, table) sql = 'truncate table {}'.format(relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod def rename(cls, profile, schema, from_name, to_name, model_name=None): - from_relation = cls.get_schema_and_table(profile, schema, from_name) + from_relation = cls.render_relation(profile, schema, from_name) sql = 'alter table {} rename to {}'.format(from_relation, to_name) connection, cursor = cls.add_query(profile, sql, model_name) @@ -636,13 +635,8 @@ def quote_schema_and_table(cls, profile, schema, table, model_name=None): cls.quote(table)) @classmethod - def get_schema_and_table(cls, profile, schema, table, quote=False, - model_name=None): - if quote: - return cls.quote_schema_and_table(profile, schema, table, - model_name) - else: - return '{}.{}'.format(schema, table) + def render_relation(cls, profile, schema, name): + return '{}.{}'.format(schema, name) @classmethod def handle_csv_table(cls, profile, schema, table_name, agate_table, diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres.py index bb1f3626886..670901f6254 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres.py @@ -199,7 +199,7 @@ def create_csv_table(cls, profile, schema, table_name, agate_table): type_ = cls.convert_agate_type(agate_table, idx) col_sqls.append('{} {}'.format(col_name, type_)) - relation = cls.get_schema_and_table(profile, schema, table_name) + relation = cls.render_relation(profile, schema, table_name) sql = 'create table {} ({})'.format(relation, ", ".join(col_sqls)) return cls.add_query(profile, sql) @@ -224,7 +224,7 @@ def load_csv_rows(cls, profile, schema, table_name, agate_table): ", ".join("%s" for _ in agate_table.column_names))) sql = ('insert into {} ({}) values {}' - .format(cls.get_schema_and_table(profile, schema, table_name), + .format(cls.render_relation(profile, schema, table_name), cols_sql, ",\n".join(placeholders))) diff --git a/dbt/utils.py b/dbt/utils.py index 2ebd0b76575..c5a37f7a189 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -50,6 +50,17 @@ def __init__(self, profile, adapter, node, use_temp=False): self.materialized = get_materialization(node) self.sql = node.get('injected_sql') + self.do_quote = self._get_quote_function(profile, adapter) + + def _get_quote_function(self, profile, adapter): + + # make a closure so we don't need to store the profile + # on the `Relation` object. That shouldn't be accessible in user-land + def quote(schema, table): + return adapter.render_relation(profile, schema, table) + + return quote + def _get_table_name(self, node): return model_immediate_name(node, dbt.flags.NON_DESTRUCTIVE) @@ -58,13 +69,13 @@ def final_name(self): msg = "final_name() was called on an ephemeral model" dbt.exceptions.raise_compiler_error(msg, self.node) else: - return "{}.{}".format(self.schema, self.table) + return self.do_quote(self.schema, self.name) def __repr__(self): if self.materialized == 'ephemeral': return '__dbt__CTE__{}'.format(self.name) else: - return "{}.{}".format(self.schema, self.table) + return self.do_quote(self.schema, self.table) def coalesce(*args): diff --git a/test/integration/022_bigquery_test/models/table_model.sql b/test/integration/022_bigquery_test/models/table_model.sql index 86b2e2f1deb..5922162c756 100644 --- a/test/integration/022_bigquery_test/models/table_model.sql +++ b/test/integration/022_bigquery_test/models/table_model.sql @@ -1,4 +1,4 @@ {{ config(materialized = "table") }} -select * from {{ ref('view') }} +select * from {{ ref('view_model') }} From 0455ea93fbe76abc74c16b5727f3918e1597e893 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sun, 1 Apr 2018 16:16:40 -0400 Subject: [PATCH 09/61] relations api --- dbt/adapters/default.py | 63 +++++++++++++++++++++++++++++++++++++++++ dbt/relation.py | 0 dbt/utils.py | 43 +--------------------------- 3 files changed, 64 insertions(+), 42 deletions(-) create mode 100644 dbt/relation.py diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index d300b00a7ee..0ad77c2b361 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -19,6 +19,69 @@ connections_available = [] +class DefaultRelation(object): + @classmethod + def create_from_node(cls, profile, adapter, node): + pass + + @classmethod + def create_from_parts(cls, database=None, schema=None, identifier=None): + return cls(database=database, schema=schema, identifier=identifier) + + @classmethod + def quote_identifier(cls, identifier): + return '"{}"'.format(identifier) + + def quote(self, database=None, schema=None, identifier=None): + new = copy.deepcopy(self) + + if database is not None: + new.should_quote_database = database + + if schema is not None: + new.should_quote_schema = schema + + if identifier is not None: + new.should_quote_identifier = identifier + + return new + + def render(self): + parts = [] + + if database is not None and self.include_database: + parts.append(self.quote_if(database, self.should_quote_database)) + + if schema is not None and self.include_schema: + parts.append(self.quote_if(schema, self.should_quote_schema)) + + parts.append(self.quote_if(identifier, self.should_quote_identifier)) + + return '.'.join(parts) + + # private + @classmethod + def quote_if(cls, identifier, should_quote): + if should_quote: + return cls.quote(identifier) + + return identifier + + def __init__(self, database=None, schema=None, identifier=None, node=None, + **kwargs): + self.database = database + self.schema = schema + self.identifier = identifier + + self.should_quote_database = False + self.should_quote_schema = False + self.should_quote_identifier = False + + self.include_database = False + self.include_schema = True + + + class DefaultAdapter(object): requires = {} diff --git a/dbt/relation.py b/dbt/relation.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbt/utils.py b/dbt/utils.py index c5a37f7a189..50d0a814a1e 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -12,6 +12,7 @@ from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_types import NodeType from dbt.clients import yaml_helper +from dbt.relation import Relation DBTConfigKeys = [ @@ -36,48 +37,6 @@ class ExitCodes(object): UnhandledError = 2 -class Relation(object): - def __init__(self, profile, adapter, node, use_temp=False): - self.node = node - self.schema = node.get('schema') - self.name = node.get('name') - - if use_temp: - self.table = self._get_table_name(node) - else: - self.table = self.name - - self.materialized = get_materialization(node) - self.sql = node.get('injected_sql') - - self.do_quote = self._get_quote_function(profile, adapter) - - def _get_quote_function(self, profile, adapter): - - # make a closure so we don't need to store the profile - # on the `Relation` object. That shouldn't be accessible in user-land - def quote(schema, table): - return adapter.render_relation(profile, schema, table) - - return quote - - def _get_table_name(self, node): - return model_immediate_name(node, dbt.flags.NON_DESTRUCTIVE) - - def final_name(self): - if self.materialized == 'ephemeral': - msg = "final_name() was called on an ephemeral model" - dbt.exceptions.raise_compiler_error(msg, self.node) - else: - return self.do_quote(self.schema, self.name) - - def __repr__(self): - if self.materialized == 'ephemeral': - return '__dbt__CTE__{}'.format(self.name) - else: - return self.do_quote(self.schema, self.table) - - def coalesce(*args): for arg in args: if arg is not None: From 75286a0d08896b4a0cba57951b02193b73ed9de9 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 2 Apr 2018 15:18:19 -0400 Subject: [PATCH 10/61] wip; good progress on Relation obj - implementing in materializations --- dbt/adapters/bigquery.py | 6 +- dbt/adapters/default.py | 92 ++-------- dbt/adapters/redshift.py | 4 +- dbt/adapters/relations/__init__.py | 0 dbt/adapters/relations/default.py | 162 ++++++++++++++++++ dbt/context/common.py | 4 +- dbt/context/parser.py | 4 +- dbt/context/runtime.py | 3 +- .../macros/materializations/bigquery.sql | 2 +- .../macros/materializations/helpers.sql | 2 +- .../macros/materializations/incremental.sql | 2 +- .../macros/materializations/table.sql | 2 +- dbt/utils.py | 1 - 13 files changed, 194 insertions(+), 90 deletions(-) create mode 100644 dbt/adapters/relations/__init__.py create mode 100644 dbt/adapters/relations/default.py diff --git a/dbt/adapters/bigquery.py b/dbt/adapters/bigquery.py index 6ec3951c01b..c847c20062f 100644 --- a/dbt/adapters/bigquery.py +++ b/dbt/adapters/bigquery.py @@ -172,12 +172,12 @@ def query_for_existing(cls, profile, schemas, model_name=None): return dict(existing) @classmethod - def drop(cls, profile, schema, relation, relation_type, model_name=None): + def drop(cls, profile, relation, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, schema, model_name) - relation_object = dataset.table(relation) + dataset = cls.get_dataset(profile, relation._schema, model_name) + relation_object = dataset.table(relation._identifier) client.delete_table(relation_object) @classmethod diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index 0ad77c2b361..b823f195fb1 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -13,74 +13,14 @@ from dbt.logger import GLOBAL_LOGGER as logger from dbt.schema import Column +import dbt.adapters.relations +import dbt.adapters.relations.default lock = multiprocessing.Lock() connections_in_use = {} connections_available = [] -class DefaultRelation(object): - @classmethod - def create_from_node(cls, profile, adapter, node): - pass - - @classmethod - def create_from_parts(cls, database=None, schema=None, identifier=None): - return cls(database=database, schema=schema, identifier=identifier) - - @classmethod - def quote_identifier(cls, identifier): - return '"{}"'.format(identifier) - - def quote(self, database=None, schema=None, identifier=None): - new = copy.deepcopy(self) - - if database is not None: - new.should_quote_database = database - - if schema is not None: - new.should_quote_schema = schema - - if identifier is not None: - new.should_quote_identifier = identifier - - return new - - def render(self): - parts = [] - - if database is not None and self.include_database: - parts.append(self.quote_if(database, self.should_quote_database)) - - if schema is not None and self.include_schema: - parts.append(self.quote_if(schema, self.should_quote_schema)) - - parts.append(self.quote_if(identifier, self.should_quote_identifier)) - - return '.'.join(parts) - - # private - @classmethod - def quote_if(cls, identifier, should_quote): - if should_quote: - return cls.quote(identifier) - - return identifier - - def __init__(self, database=None, schema=None, identifier=None, node=None, - **kwargs): - self.database = database - self.schema = schema - self.identifier = identifier - - self.should_quote_database = False - self.should_quote_schema = False - self.should_quote_identifier = False - - self.include_database = False - self.include_schema = True - - class DefaultAdapter(object): @@ -106,6 +46,8 @@ class DefaultAdapter(object): "quote", ] + Relation = dbt.adapters.relations.default.DefaultRelation + ### # ADAPTER-SPECIFIC FUNCTIONS -- each of these must be overridden in # every adapter @@ -190,30 +132,28 @@ def get_result_from_cursor(cls, cursor): return dbt.clients.agate_helper.table_from_data(data) @classmethod - def drop(cls, profile, schema, relation, relation_type, model_name=None): - if relation_type == 'view': - return cls.drop_view(profile, schema, relation, model_name) - elif relation_type == 'table': - return cls.drop_table(profile, schema, relation, model_name) + def drop(cls, profile, relation, model_name=None): + if relation._type == 'view': + return cls.drop_view(profile, relation, model_name) + elif relation._type == 'table': + return cls.drop_table(profile, relation, model_name) else: raise RuntimeError( "Invalid relation_type '{}'" - .format(relation_type)) + .format(relation._type)) @classmethod - def drop_relation(cls, profile, schema, rel_name, rel_type, model_name): - relation = cls.render_relation(profile, schema, rel_name) - sql = 'drop {} if exists {} cascade'.format(rel_type, relation) - + def drop_relation(cls, profile, relation, model_name): + sql = 'drop {} if exists {} cascade'.format(relation._type, relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def drop_view(cls, profile, schema, view, model_name): - cls.drop_relation(profile, schema, view, 'view', model_name) + def drop_view(cls, profile, relation, model_name): + cls.drop_relation(profile, relation, model_name) @classmethod - def drop_table(cls, profile, schema, table, model_name): - cls.drop_relation(profile, schema, table, 'table', model_name) + def drop_table(cls, profile, relation, model_name): + cls.drop_relation(profile, relation, model_name) @classmethod def truncate(cls, profile, schema, table, model_name=None): diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift.py index d3009e3e5ce..d16a9a15150 100644 --- a/dbt/adapters/redshift.py +++ b/dbt/adapters/redshift.py @@ -69,7 +69,7 @@ def _get_columns_in_table_sql(cls, schema_name, table_name): return sql @classmethod - def drop(cls, profile, schema, relation, relation_type, model_name=None): + def drop(cls, profile, relation, model_name=None): global drop_lock to_return = None @@ -85,7 +85,7 @@ def drop(cls, profile, schema, relation, relation_type, model_name=None): cls.begin(profile, connection.get('name')) to_return = super(PostgresAdapter, cls).drop( - profile, schema, relation, relation_type, model_name) + profile, relation, model_name) cls.commit(profile, connection) cls.begin(profile, connection.get('name')) diff --git a/dbt/adapters/relations/__init__.py b/dbt/adapters/relations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbt/adapters/relations/default.py b/dbt/adapters/relations/default.py new file mode 100644 index 00000000000..a74e9201550 --- /dev/null +++ b/dbt/adapters/relations/default.py @@ -0,0 +1,162 @@ + + +# This should implement schema/name props for compatibility with old relation obj +class DefaultRelation(object): + QuoteCharacter = '"' + QuotePolicy = { + "database": True, + "schema": True, + "identifier": True + } + + IncludePolicy = { + "database": False, + "schema": True, + "identifier": True + } + + Table = "table" + View = "view" + + RelationTypes = [ + Table, + View + ] + + @property + def database(self): + return self._database + + @property + def schema(self): + return self._schema + + @property + def identifier(self): + return self._identifier + + # Here for compatibility with old Relation interface + @property + def name(self): + return self.identifier + + def should_quote(self, part): + return self._quoting.get(part) + + def should_include(self, part): + return self._include.get(part) + + @property + def inclusion(self): + return self._include + + @property + def quoting(self): + return self._quoting + + @classmethod + def create_from_node(cls, profile, adapter, node): + import ipdb; ipdb.set_trace() + pass + + @classmethod + def create_from_parts(cls, database=None, schema=None, identifier=None): + return cls(database=database, schema=schema, identifier=identifier) + + def quote(self, database=None, schema=None, identifier=None): + raw_policy = { + "database": database, + "schema": schema, + "identifier": identifier + } + + policy_update = {k: v for (k,v) in raw_policy.items() if v is not None} + policy = self.quoting.copy() + policy.update(policy_update) + + return type(self)(self._type, + database=self.database, + schema=self.schema, + identifier=self.identifier, + quoting=policy, + include=self.inclusion) + + def include(self, database=None, schema=None, identifier=None): + raw_policy = { + "database": database, + "schema": schema, + "identifier": identifier + } + + policy_update = {k: v for (k,v) in raw_policy.items() if v is not None} + policy = self.inclusion.copy() + policy.update(policy_update) + + return type(self)(self._type, + database=self.database, + schema=self.schema, + identifier=self.identifier, + quoting=self.quoting, + include=policy) + + def render(self): + parts = [] + + if self.database is not None and self.should_include('database'): + parts.append(self.quote_if(self.database, self.should_quote('database'))) + + if self.schema is not None and self.should_include('schema'): + parts.append(self.quote_if(self.schema, self.should_quote('schema'))) + + if self.identifier is not None and self.should_include('identifier'): + parts.append(self.quote_if(self.identifier, self.should_quote('identifier'))) + + + if len(parts) == 0: + # TODO + raise RuntimeError("Nothing to quote here....") + + return '.'.join(parts) + + def quote_if(self, identifier, should_quote): + if should_quote: + return self.quoted(identifier) + + return identifier + + @classmethod + def quoted(cls, s): + return '{quote_char}{identifier}{quote_char}'.format( + quote_char=cls.QuoteCharacter, identifier=s) + + def __init__(self, relation_type=None, database=None, schema=None, + identifier=None, quoting=None, include=None): + + if relation_type not in DefaultRelation.RelationTypes: + # TODO - compiler error + raise RuntimeError("Relation Type {} is invalid".format( + relation_type)) + + self._type = relation_type + + self._database = database + self._schema = schema + self._identifier = identifier + self._type = relation_type + + self._quoting = self.QuotePolicy.copy() + self._quoting.update(quoting or {}) + + self._include = self.IncludePolicy.copy() + self._include.update(include or {}) + + def __repr__(self): + return "<{} {}: {}>".format(self.__class__.__name__, self._type, self.render()) + + def __str__(self): + return self.render() + + +if __name__ == '__main__': + r = DefaultRelation("table", database='whatever', schema='ok', identifier='cool') + r.render() diff --git a/dbt/context/common.py b/dbt/context/common.py index 97188eabea6..d5a225428de 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -300,6 +300,7 @@ def generate(model, project, flat_graph, provider=None): context = dbt.utils.merge(context, { "adapter": db_wrapper, + "Relation": db_wrapper.adapter.Relation, "column": dbt.schema.Column, "config": provider.Config(model), "env_var": _env_var, @@ -323,7 +324,8 @@ def generate(model, project, flat_graph, provider=None): "fromjson": fromjson, "tojson": tojson, "target": target, - "this": dbt.utils.Relation(profile, adapter, model, use_temp=True), + # TODO!! + "this": db_wrapper.adapter.Relation('table', schema=model['schema'], identifier="{}__dbt_tmp".format(model['name'])), "try_or_compiler_error": try_or_compiler_error(model) }) diff --git a/dbt/context/parser.py b/dbt/context/parser.py index bf254bb3636..788711ddd6b 100644 --- a/dbt/context/parser.py +++ b/dbt/context/parser.py @@ -1,4 +1,3 @@ -import dbt.utils import dbt.exceptions import dbt.context.common @@ -19,7 +18,8 @@ def ref(*args): dbt.exceptions.ref_invalid_args(model, args) adapter = get_adapter(profile) - return dbt.utils.Relation(profile, adapter, model) + #TODO + return adapter.Relation('table', schema=model['schema'], identifier=model['name']) return ref diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index 1a67ab1f5b1..e70e5edd930 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -52,7 +52,8 @@ def do_ref(*args): model['extra_ctes'][target_model_id] = None adapter = get_adapter(profile) - return dbt.utils.Relation(profile, adapter, target_model) + #TODO !!!! + return adapter.Relation('table', schema=target_model['schema'], identifier=target_model['name']) return do_ref diff --git a/dbt/include/global_project/macros/materializations/bigquery.sql b/dbt/include/global_project/macros/materializations/bigquery.sql index 7762d2b1863..9f1a3d65d86 100644 --- a/dbt/include/global_project/macros/materializations/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/bigquery.sql @@ -77,7 +77,7 @@ if it is not a table. If it _is_ already a table, then we can overwrite it without downtime #} {%- if existing_type is not none and existing_type != 'table' -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {{ adapter.drop(Relation(existing_type, schema=schema, identifier=identifier)) }} {%- endif -%} -- build model diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 3eb190e3e08..7727d06bda0 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -49,6 +49,6 @@ {% macro drop_if_exists(existing, schema, name) %} {% set existing_type = get_existing_relation_type(existing, name) %} {% if existing_type is not none %} - {{ adapter.drop(schema, name, existing_type) }} + {{ adapter.drop(Relation(existing_type, schema=schema, identifier=name)) }} {% endif %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index b3260a6ea84..66436eb8f1c 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -35,7 +35,7 @@ {%- elif should_truncate -%} {{ adapter.truncate(schema, identifier) }} {%- elif should_drop -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {{ adapter.drop(Relation(existing_type, schema=schema, identifier=identifier)) }} {%- endif %} {{ run_hooks(pre_hooks, inside_transaction=False) }} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index c646bb68c3a..7d3271bbf73 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -12,7 +12,7 @@ {% if existing_type == 'table' -%} {{ adapter.truncate(schema, identifier) }} {% elif existing_type == 'view' -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {{ adapter.drop(Relation(existing_type, schema=schema, identifier=identifier)) }} {%- endif %} {%- endif %} diff --git a/dbt/utils.py b/dbt/utils.py index 50d0a814a1e..c423bb8335c 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -12,7 +12,6 @@ from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_types import NodeType from dbt.clients import yaml_helper -from dbt.relation import Relation DBTConfigKeys = [ From c7062639b205bf8c4c00f7659ec069b580dd244e Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 2 Apr 2018 18:00:32 -0400 Subject: [PATCH 11/61] hack for ephemerals; dont include type --- dbt/adapters/default.py | 16 +++---- dbt/adapters/redshift.py | 2 +- dbt/adapters/relations/default.py | 48 +++++++++++-------- dbt/adapters/relations/ephemeral.py | 8 ++++ dbt/context/common.py | 12 +++-- dbt/context/parser.py | 3 +- dbt/context/runtime.py | 13 +++-- .../macros/materializations/bigquery.sql | 4 +- .../macros/materializations/helpers.sql | 2 +- .../macros/materializations/incremental.sql | 2 +- .../macros/materializations/table.sql | 2 +- 11 files changed, 68 insertions(+), 44 deletions(-) create mode 100644 dbt/adapters/relations/ephemeral.py diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index b823f195fb1..8310dca14b1 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -132,28 +132,28 @@ def get_result_from_cursor(cls, cursor): return dbt.clients.agate_helper.table_from_data(data) @classmethod - def drop(cls, profile, relation, model_name=None): - if relation._type == 'view': + def drop(cls, profile, rel_type, relation, model_name=None): + if rel_type == 'view': return cls.drop_view(profile, relation, model_name) - elif relation._type == 'table': + elif rel_type == 'table': return cls.drop_table(profile, relation, model_name) else: raise RuntimeError( "Invalid relation_type '{}'" - .format(relation._type)) + .format(rel_type)) @classmethod - def drop_relation(cls, profile, relation, model_name): - sql = 'drop {} if exists {} cascade'.format(relation._type, relation) + def drop_relation(cls, profile, rel_type, relation, model_name): + sql = 'drop {} if exists {} cascade'.format(rel_type, relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod def drop_view(cls, profile, relation, model_name): - cls.drop_relation(profile, relation, model_name) + cls.drop_relation(profile, 'view', relation, model_name) @classmethod def drop_table(cls, profile, relation, model_name): - cls.drop_relation(profile, relation, model_name) + cls.drop_relation(profile, 'table', relation, model_name) @classmethod def truncate(cls, profile, schema, table, model_name=None): diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift.py index d16a9a15150..a8ecb661657 100644 --- a/dbt/adapters/redshift.py +++ b/dbt/adapters/redshift.py @@ -85,7 +85,7 @@ def drop(cls, profile, relation, model_name=None): cls.begin(profile, connection.get('name')) to_return = super(PostgresAdapter, cls).drop( - profile, relation, model_name) + profile, 'table', relation, model_name) cls.commit(profile, connection) cls.begin(profile, connection.get('name')) diff --git a/dbt/adapters/relations/default.py b/dbt/adapters/relations/default.py index a74e9201550..842f4b57f83 100644 --- a/dbt/adapters/relations/default.py +++ b/dbt/adapters/relations/default.py @@ -40,6 +40,11 @@ def identifier(self): def name(self): return self.identifier + # Here for compatibility with old Relation interface + @property + def table(self): + return self._table_name + def should_quote(self, part): return self._quoting.get(part) @@ -54,15 +59,6 @@ def inclusion(self): def quoting(self): return self._quoting - @classmethod - def create_from_node(cls, profile, adapter, node): - import ipdb; ipdb.set_trace() - pass - - @classmethod - def create_from_parts(cls, database=None, schema=None, identifier=None): - return cls(database=database, schema=schema, identifier=identifier) - def quote(self, database=None, schema=None, identifier=None): raw_policy = { "database": database, @@ -74,7 +70,7 @@ def quote(self, database=None, schema=None, identifier=None): policy = self.quoting.copy() policy.update(policy_update) - return type(self)(self._type, + return type(self)( database=self.database, schema=self.schema, identifier=self.identifier, @@ -92,7 +88,7 @@ def include(self, database=None, schema=None, identifier=None): policy = self.inclusion.copy() policy.update(policy_update) - return type(self)(self._type, + return type(self)( database=self.database, schema=self.schema, identifier=self.identifier, @@ -129,20 +125,32 @@ def quoted(cls, s): return '{quote_char}{identifier}{quote_char}'.format( quote_char=cls.QuoteCharacter, identifier=s) - def __init__(self, relation_type=None, database=None, schema=None, - identifier=None, quoting=None, include=None): + @classmethod + def create_from_node(cls, profile, node, **kwargs): + return cls( + database=profile['dbname'], + schema=node['schema'], + identifier=node['name'], + **kwargs + ) - if relation_type not in DefaultRelation.RelationTypes: - # TODO - compiler error - raise RuntimeError("Relation Type {} is invalid".format( - relation_type)) + @classmethod + def create_from_parts(cls, database=None, schema=None, identifier=None): + return cls(database=database, schema=schema, identifier=identifier) - self._type = relation_type + def __init__(self, database=None, schema=None, identifier=None, + table_name=None, quoting=None, include=None): self._database = database self._schema = schema self._identifier = identifier - self._type = relation_type + + # This field is deprecated, but exists for backwards compatibility + # with the existing implementation of Relations + if table_name is None: + self._table_name = identifier + else: + self._table_name = table_name self._quoting = self.QuotePolicy.copy() self._quoting.update(quoting or {}) @@ -151,7 +159,7 @@ def __init__(self, relation_type=None, database=None, schema=None, self._include.update(include or {}) def __repr__(self): - return "<{} {}: {}>".format(self.__class__.__name__, self._type, self.render()) + return "<{} {}>".format(self.__class__.__name__, self.render()) def __str__(self): return self.render() diff --git a/dbt/adapters/relations/ephemeral.py b/dbt/adapters/relations/ephemeral.py new file mode 100644 index 00000000000..d2cedb9d0c1 --- /dev/null +++ b/dbt/adapters/relations/ephemeral.py @@ -0,0 +1,8 @@ + + +from dbt.adapters.relations.default import DefaultRelation + + +class EphemeralRelation(DefaultRelation): + def render(self): + return '__dbt__CTE__{}'.format(self.identifier) diff --git a/dbt/context/common.py b/dbt/context/common.py index d5a225428de..f048d44027a 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -269,7 +269,14 @@ def impl(message_if_exception, func, *args, **kwargs): def _return(value): - raise dbt.exceptions.MacroReturn(value) + raise dbt.exceptions.MacroReturn(value) + + +def get_this_relation(db_wrapper, profile, model): + table_name = dbt.utils.model_immediate_name( + model, dbt.flags.NON_DESTRUCTIVE) + return db_wrapper.adapter.Relation.create_from_node(profile, model, + table_name=table_name) def generate(model, project, flat_graph, provider=None): @@ -324,8 +331,7 @@ def generate(model, project, flat_graph, provider=None): "fromjson": fromjson, "tojson": tojson, "target": target, - # TODO!! - "this": db_wrapper.adapter.Relation('table', schema=model['schema'], identifier="{}__dbt_tmp".format(model['name'])), + "this": get_this_relation(db_wrapper, profile, model), "try_or_compiler_error": try_or_compiler_error(model) }) diff --git a/dbt/context/parser.py b/dbt/context/parser.py index 788711ddd6b..3cce11d298d 100644 --- a/dbt/context/parser.py +++ b/dbt/context/parser.py @@ -18,8 +18,7 @@ def ref(*args): dbt.exceptions.ref_invalid_args(model, args) adapter = get_adapter(profile) - #TODO - return adapter.Relation('table', schema=model['schema'], identifier=model['name']) + return adapter.Relation.create_from_node(profile, model) return ref diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index e70e5edd930..8a89de15246 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -8,6 +8,8 @@ import dbt.flags import dbt.utils +from dbt.adapters.relations.ephemeral import EphemeralRelation + from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -48,12 +50,13 @@ def do_ref(*args): target_model_name, target_model_package) - if dbt.utils.get_materialization(target_model) == 'ephemeral': + is_ephemeral = dbt.utils.get_materialization(target_model) == 'ephemeral' + if is_ephemeral: model['extra_ctes'][target_model_id] = None - - adapter = get_adapter(profile) - #TODO !!!! - return adapter.Relation('table', schema=target_model['schema'], identifier=target_model['name']) + return EphemeralRelation.create_from_node(profile, target_model) + else: + adapter = get_adapter(profile) + return adapter.Relation.create_from_node(profile, target_model) return do_ref diff --git a/dbt/include/global_project/macros/materializations/bigquery.sql b/dbt/include/global_project/macros/materializations/bigquery.sql index 9f1a3d65d86..d772ff3b7d4 100644 --- a/dbt/include/global_project/macros/materializations/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/bigquery.sql @@ -15,7 +15,7 @@ {{ exceptions.raise_compiler_error(error_message) }} {%- endif -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} {%- endif -%} -- build model @@ -77,7 +77,7 @@ if it is not a table. If it _is_ already a table, then we can overwrite it without downtime #} {%- if existing_type is not none and existing_type != 'table' -%} - {{ adapter.drop(Relation(existing_type, schema=schema, identifier=identifier)) }} + {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} {%- endif -%} -- build model diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 7727d06bda0..5b53879e6fd 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -49,6 +49,6 @@ {% macro drop_if_exists(existing, schema, name) %} {% set existing_type = get_existing_relation_type(existing, name) %} {% if existing_type is not none %} - {{ adapter.drop(Relation(existing_type, schema=schema, identifier=name)) }} + {{ adapter.drop(existing_type, Relation(schema=schema, identifier=name)) }} {% endif %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index 66436eb8f1c..dc061742747 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -35,7 +35,7 @@ {%- elif should_truncate -%} {{ adapter.truncate(schema, identifier) }} {%- elif should_drop -%} - {{ adapter.drop(Relation(existing_type, schema=schema, identifier=identifier)) }} + {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} {%- endif %} {{ run_hooks(pre_hooks, inside_transaction=False) }} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 7d3271bbf73..3bb7c01b132 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -12,7 +12,7 @@ {% if existing_type == 'table' -%} {{ adapter.truncate(schema, identifier) }} {% elif existing_type == 'view' -%} - {{ adapter.drop(Relation(existing_type, schema=schema, identifier=identifier)) }} + {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} {%- endif %} {%- endif %} From aae580ea285c233866f6f2ae3a809c513ce87c60 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 4 Apr 2018 13:56:42 -0400 Subject: [PATCH 12/61] wiring up Relation to adapter fns, first pass --- dbt/adapters/bigquery.py | 33 +++++--- dbt/adapters/default.py | 84 ++++++++++++++----- dbt/adapters/postgres.py | 25 +++--- dbt/adapters/relations/default.py | 46 ++++++++-- dbt/deprecations.py | 5 ++ dbt/exceptions.py | 8 ++ .../macros/materializations/bigquery.sql | 4 +- .../macros/materializations/helpers.sql | 2 +- .../macros/materializations/incremental.sql | 2 +- .../macros/materializations/table.sql | 2 +- 10 files changed, 150 insertions(+), 61 deletions(-) diff --git a/dbt/adapters/bigquery.py b/dbt/adapters/bigquery.py index c847c20062f..9ec3ab7bade 100644 --- a/dbt/adapters/bigquery.py +++ b/dbt/adapters/bigquery.py @@ -27,6 +27,7 @@ class BigQueryAdapter(PostgresAdapter): "query_for_existing", "execute_model", "drop", + "drop_relation", "execute", "quote_schema_and_table", "make_date_partitioned_table" @@ -152,13 +153,23 @@ def query_for_existing(cls, profile, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): schemas = [schemas] - conn = cls.get_connection(profile, model_name) - client = conn.get('handle') + all_relations = [] - all_tables = [] for schema in schemas: - dataset = cls.get_dataset(profile, schema, model_name) - all_tables.extend(client.list_tables(dataset)) + all_relations.extend(cls.list_relations( + profile, schema, model_name)) + + return {relation.identifier: relation.type + for relation in all_relations} + + @classmethod + def list_relations(cls, profile, dataset, model_name=None): + connection = cls.get_connection(profile, model_name) + credentials = connection.get('credentials', {}) + client = connection.get('handle') + + bigquery_dataset = cls.get_dataset(profile, dataset, model_name) + all_tables = client.list_tables(bigquery_dataset) relation_types = { 'TABLE': 'table', @@ -166,13 +177,15 @@ def query_for_existing(cls, profile, schemas, model_name=None): 'EXTERNAL': 'external' } - existing = [(table.table_id, relation_types.get(table.table_type)) - for table in all_tables] - - return dict(existing) + return [cls.Relation.create_from_parts( + project=credentials.get('project'), + dataset=dataset, + identfier=table.table_id, + type=relation_types.get(table.table_type)) + for table in all_tables] @classmethod - def drop(cls, profile, relation, model_name=None): + def drop_relation(cls, profile, relation, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index 8310dca14b1..866b3d6eb01 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -21,12 +21,12 @@ connections_available = [] - class DefaultAdapter(object): requires = {} context_functions = [ + # deprecated -- use versions that take relations instead "already_exists", "get_columns_in_table", "get_missing_columns", @@ -34,10 +34,19 @@ class DefaultAdapter(object): "rename", "drop", "truncate", - "add_query", "expand_target_column_types", + + # just deprecated. going away in a future release "quote_schema_and_table", - "execute" + + # versions of adapter functions that take / return Relations + "list_relations", + "get_relation", + "drop_relation", + + + "execute", + "add_query", ] raw_functions = [ @@ -82,8 +91,14 @@ def alter_column_type(cls, profile, schema, table, column_name, @classmethod def query_for_existing(cls, profile, schemas, model_name=None): - raise dbt.exceptions.NotImplementedException( - '`query_for_existing` is not implemented for this adapter!') + if not isinstance(schemas, (list, tuple)): + schemas = [schemas] + + all_relations = cls.list_relations(profile, model_name) + + return {relation.identifier: relation.type + for relation in all_relations + if relation.schema in schemas} @classmethod def get_existing_schemas(cls, profile, model_name=None): @@ -132,28 +147,20 @@ def get_result_from_cursor(cls, cursor): return dbt.clients.agate_helper.table_from_data(data) @classmethod - def drop(cls, profile, rel_type, relation, model_name=None): - if rel_type == 'view': - return cls.drop_view(profile, relation, model_name) - elif rel_type == 'table': - return cls.drop_table(profile, relation, model_name) - else: - raise RuntimeError( - "Invalid relation_type '{}'" - .format(rel_type)) + def drop(cls, profile, schema, relation, relation_type, model_name=None): + # add deprecation warning + identifier = relation + relation = cls.Relation(schema=schema, + identifier=identifier, + type=relation_type) - @classmethod - def drop_relation(cls, profile, rel_type, relation, model_name): - sql = 'drop {} if exists {} cascade'.format(rel_type, relation) - connection, cursor = cls.add_query(profile, sql, model_name) + return cls.drop_relation(profile, relation, model_name) @classmethod - def drop_view(cls, profile, relation, model_name): - cls.drop_relation(profile, 'view', relation, model_name) + def drop_relation(cls, profile, relation, model_name=None): + sql = 'drop {} if exists {} cascade'.format(relation.type, relation) - @classmethod - def drop_table(cls, profile, relation, model_name): - cls.drop_relation(profile, 'table', relation, model_name) + connection, cursor = cls.add_query(profile, sql, model_name) @classmethod def truncate(cls, profile, schema, table, model_name=None): @@ -257,6 +264,37 @@ def expand_target_column_types(cls, profile, cls.alter_column_type(profile, to_schema, to_table, column_name, new_type, model_name) + ### + # RELATIONS + ### + @classmethod + def list_relations(cls, profile, model_name=None): + raise dbt.exceptions.NotImplementedException( + '`list_relations` is not implemented for this adapter!') + + @classmethod + def get_relation(cls, profile, **kwargs): + # make sure that if this returns multiple relations we do + # something smart + # > adapter.get_relation('orders') -- orders exists in two schemas + relations = cls.list_relations() + + matches = [] + + for relation in relations: + if relation.matches(**kwargs): + matches.append(relation) + + if len(matches) > 1: + dbt.exceptions.get_relation_returned_multiple_results( + kwargs, matches) + + elif matches: + return matches[0] + + return None + + ### # SANE ANSI SQL DEFAULTS ### diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres.py index 670901f6254..0d6d8b2d5bb 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres.py @@ -108,28 +108,23 @@ def alter_column_type(cls, profile, schema, table, column_name, return connection, cursor @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): - if not isinstance(schemas, (list, tuple)): - schemas = [schemas] - - schema_list = ",".join(["'{}'".format(schema) for schema in schemas]) - + def list_relations(cls, profile, model_name=None): sql = """ - select tablename as name, 'table' as type from pg_tables - where schemaname in ({schema_list}) + select tablename as name, schemaname as schema, 'table' as type from pg_tables union all - select viewname as name, 'view' as type from pg_views - where schemaname in ({schema_list}) - """.format(schema_list=schema_list).strip() # noqa + select viewname as name, schemaname as schema, 'view' as type from pg_views + """.strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name, auto_begin=False) results = cursor.fetchall() - existing = [(name, relation_type) for (name, relation_type) in results] - - return dict(existing) + return [cls.Relation.create_from_parts(database=profile.get('dbname'), + schema=schema, + identifier=name, + type=type) + for (name, schema, type) in results] @classmethod def get_existing_schemas(cls, profile, model_name=None): @@ -207,7 +202,7 @@ def create_csv_table(cls, profile, schema, table_name, agate_table): def reset_csv_table(cls, profile, schema, table_name, agate_table, full_refresh=False): if full_refresh: - cls.drop_table(profile, schema, table_name, None) + cls.drop(profile, schema, table_name, 'table', None) cls.create_csv_table(profile, schema, table_name, agate_table) else: cls.truncate(profile, schema, table_name) diff --git a/dbt/adapters/relations/default.py b/dbt/adapters/relations/default.py index 842f4b57f83..ba9e317d114 100644 --- a/dbt/adapters/relations/default.py +++ b/dbt/adapters/relations/default.py @@ -1,6 +1,8 @@ +from dbt.utils import merge -# This should implement schema/name props for compatibility with old relation obj +# This should implement schema/name props for compatibility with old relation +# obj class DefaultRelation(object): QuoteCharacter = '"' QuotePolicy = { @@ -23,10 +25,26 @@ class DefaultRelation(object): View ] + def matches(self, database=None, schema=None, identifier=None): + if database is not None and database != self.database: + return False + + if schema is not None and schema != self.schema: + return False + + if identifier is not None and identifier != self.identifier: + return False + + return True + @property def database(self): return self._database + @property + def type(self): + return self._type + @property def schema(self): return self._schema @@ -45,6 +63,14 @@ def name(self): def table(self): return self._table_name + @property + def is_table(self): + return self.type == self.Table + + @property + def is_view(self): + return self.type == self.View + def should_quote(self, part): return self._quoting.get(part) @@ -83,10 +109,11 @@ def include(self, database=None, schema=None, identifier=None): "schema": schema, "identifier": identifier } + raw_policy = {k: v for (k, v) in raw_policy.items() if v is not None} - policy_update = {k: v for (k,v) in raw_policy.items() if v is not None} - policy = self.inclusion.copy() - policy.update(policy_update) + policy = merge( + self.inclusion, + raw_policy) return type(self)( database=self.database, @@ -107,7 +134,6 @@ def render(self): if self.identifier is not None and self.should_include('identifier'): parts.append(self.quote_if(self.identifier, self.should_quote('identifier'))) - if len(parts) == 0: # TODO raise RuntimeError("Nothing to quote here....") @@ -135,15 +161,17 @@ def create_from_node(cls, profile, node, **kwargs): ) @classmethod - def create_from_parts(cls, database=None, schema=None, identifier=None): - return cls(database=database, schema=schema, identifier=identifier) + def create_from_parts(cls, **kwargs): + # just use constructor + return cls(**kwargs) def __init__(self, database=None, schema=None, identifier=None, - table_name=None, quoting=None, include=None): + table_name=None, type=None, quoting=None, include=None): self._database = database self._schema = schema self._identifier = identifier + self._type = type # This field is deprecated, but exists for backwards compatibility # with the existing implementation of Relations @@ -158,6 +186,8 @@ def __init__(self, database=None, schema=None, identifier=None, self._include = self.IncludePolicy.copy() self._include.update(include or {}) + # validate + def __repr__(self): return "<{} {}>".format(self.__class__.__name__, self.render()) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index 00d656b7be0..aa0e36b3309 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -33,6 +33,11 @@ class SeedDropExistingDeprecation(DBTDeprecation): will be removed in a future version of dbt.""" +class OldStyleAdapterFunctionsDeprecations(DBTDeprecation): + name = 'old-style-adapter-functions' + description = """Use the new version of this thing?""" + + def warn(name, *args, **kwargs): if name not in deprecations: # this should (hopefully) never happen diff --git a/dbt/exceptions.py b/dbt/exceptions.py index e6eff4b36d9..94d72b7f497 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -291,3 +291,11 @@ def raise_dep_not_found(node, node_description, required_pkg): 'Error while parsing {}.\nThe required package "{}" was not found. ' 'Is the package installed?\nHint: You may need to run ' '`dbt deps`.'.format(node_description, required_pkg), node=node) + + +def get_relation_returned_multiple_results(kwargs, matches): + raise_compiler_error( + 'get_relation returned more than one relation with the given args. ' + 'Please specify a database or schema to narrow down the result set.' + '\n{}\n\n{}' + .format(kwargs, matches)) diff --git a/dbt/include/global_project/macros/materializations/bigquery.sql b/dbt/include/global_project/macros/materializations/bigquery.sql index d772ff3b7d4..bb12044ccba 100644 --- a/dbt/include/global_project/macros/materializations/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/bigquery.sql @@ -15,7 +15,7 @@ {{ exceptions.raise_compiler_error(error_message) }} {%- endif -%} - {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} + {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} {%- endif -%} -- build model @@ -77,7 +77,7 @@ if it is not a table. If it _is_ already a table, then we can overwrite it without downtime #} {%- if existing_type is not none and existing_type != 'table' -%} - {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} + {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} {%- endif -%} -- build model diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 5b53879e6fd..cc7bb97168c 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -49,6 +49,6 @@ {% macro drop_if_exists(existing, schema, name) %} {% set existing_type = get_existing_relation_type(existing, name) %} {% if existing_type is not none %} - {{ adapter.drop(existing_type, Relation(schema=schema, identifier=name)) }} + {{ adapter.drop_relation(Relation(schema=schema, identifier=name, type=existing_type)) }} {% endif %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index dc061742747..3b6991d0906 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -35,7 +35,7 @@ {%- elif should_truncate -%} {{ adapter.truncate(schema, identifier) }} {%- elif should_drop -%} - {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} + {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} {%- endif %} {{ run_hooks(pre_hooks, inside_transaction=False) }} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 3bb7c01b132..19938df2572 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -12,7 +12,7 @@ {% if existing_type == 'table' -%} {{ adapter.truncate(schema, identifier) }} {% elif existing_type == 'view' -%} - {{ adapter.drop(existing_type, Relation(schema=schema, identifier=identifier)) }} + {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} {%- endif %} {%- endif %} From e0153384e1831e829b09885caac66dcfa126b8c9 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 5 Apr 2018 15:29:29 -0400 Subject: [PATCH 13/61] code moved around, defaultrelation written, starting to wire up --- Makefile | 2 +- TODO.md | 7 + dbt/adapters/bigquery/__init__.py | 5 + .../{bigquery.py => bigquery/impl.py} | 4 +- dbt/adapters/default/__init__.py | 7 + dbt/adapters/{default.py => default/impl.py} | 8 +- dbt/adapters/default/relation.py | 214 ++++++++++++++++++ dbt/adapters/postgres/__init__.py | 5 + .../{postgres.py => postgres/impl.py} | 8 +- .../{redshift.py => redshift/__init__.py} | 0 dbt/adapters/relations/default.py | 200 ---------------- dbt/adapters/snowflake/__init__.py | 5 + .../{snowflake.py => snowflake/impl.py} | 0 dbt/api/__init__.py | 5 + dbt/api/object.py | 82 +++++++ .../macros/materializations/bigquery.sql | 18 +- dbt/utils.py | 5 + dbt_project.yml | 2 + setup.py | 1 + 19 files changed, 355 insertions(+), 223 deletions(-) create mode 100644 TODO.md create mode 100644 dbt/adapters/bigquery/__init__.py rename dbt/adapters/{bigquery.py => bigquery/impl.py} (99%) create mode 100644 dbt/adapters/default/__init__.py rename dbt/adapters/{default.py => default/impl.py} (98%) create mode 100644 dbt/adapters/default/relation.py create mode 100644 dbt/adapters/postgres/__init__.py rename dbt/adapters/{postgres.py => postgres/impl.py} (96%) rename dbt/adapters/{redshift.py => redshift/__init__.py} (100%) delete mode 100644 dbt/adapters/relations/default.py create mode 100644 dbt/adapters/snowflake/__init__.py rename dbt/adapters/{snowflake.py => snowflake/impl.py} (100%) create mode 100644 dbt/api/__init__.py create mode 100644 dbt/api/object.py create mode 100644 dbt_project.yml diff --git a/Makefile b/Makefile index ca6a0dfe085..3990df87e57 100644 --- a/Makefile +++ b/Makefile @@ -20,4 +20,4 @@ test-integration: test-quick: @echo "Integration test run starting..." - @time docker-compose run test tox -e integration-snowflake-py27 -- -x + @time docker-compose run test tox -e integration-postgres-py36 -- -x diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000000..b5163ec0422 --- /dev/null +++ b/TODO.md @@ -0,0 +1,7 @@ +- [ ] rip out query_for_existing > replace with get_relation + - maybe part of the relation object is metadata about when the relation was pulled + from the db (relation type, when we asked the db about it, columns, ??) +- [ ] add get_relation, list_relations +- fns / macros + - query_for_existing + - macro: get_existing_relation_type diff --git a/dbt/adapters/bigquery/__init__.py b/dbt/adapters/bigquery/__init__.py new file mode 100644 index 00000000000..6ff9bcccd3f --- /dev/null +++ b/dbt/adapters/bigquery/__init__.py @@ -0,0 +1,5 @@ +from dbt.adapters.bigquery.impl import BigqueryAdapter + +__all__ = [ + BigqueryAdapter, +] diff --git a/dbt/adapters/bigquery.py b/dbt/adapters/bigquery/impl.py similarity index 99% rename from dbt/adapters/bigquery.py rename to dbt/adapters/bigquery/impl.py index 9ec3ab7bade..dc3194aadb2 100644 --- a/dbt/adapters/bigquery.py +++ b/dbt/adapters/bigquery/impl.py @@ -177,10 +177,10 @@ def list_relations(cls, profile, dataset, model_name=None): 'EXTERNAL': 'external' } - return [cls.Relation.create_from_parts( + return [cls.Relation( project=credentials.get('project'), dataset=dataset, - identfier=table.table_id, + identifier=table.table_id, type=relation_types.get(table.table_type)) for table in all_tables] diff --git a/dbt/adapters/default/__init__.py b/dbt/adapters/default/__init__.py new file mode 100644 index 00000000000..c56e8b54380 --- /dev/null +++ b/dbt/adapters/default/__init__.py @@ -0,0 +1,7 @@ +from dbt.adapters.default.impl import DefaultAdapter +from dbt.adapters.default.relation import DefaultRelation + +__all__ = [ + DefaultAdapter, + DefaultRelation, +] diff --git a/dbt/adapters/default.py b/dbt/adapters/default/impl.py similarity index 98% rename from dbt/adapters/default.py rename to dbt/adapters/default/impl.py index 866b3d6eb01..200f763431f 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default/impl.py @@ -13,8 +13,7 @@ from dbt.logger import GLOBAL_LOGGER as logger from dbt.schema import Column -import dbt.adapters.relations -import dbt.adapters.relations.default +from dbt.adapters.default.relation import DefaultRelation lock = multiprocessing.Lock() connections_in_use = {} @@ -55,7 +54,7 @@ class DefaultAdapter(object): "quote", ] - Relation = dbt.adapters.relations.default.DefaultRelation + Relation = DefaultRelation ### # ADAPTER-SPECIFIC FUNCTIONS -- each of these must be overridden in @@ -274,9 +273,6 @@ def list_relations(cls, profile, model_name=None): @classmethod def get_relation(cls, profile, **kwargs): - # make sure that if this returns multiple relations we do - # something smart - # > adapter.get_relation('orders') -- orders exists in two schemas relations = cls.list_relations() matches = [] diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py new file mode 100644 index 00000000000..2ba0e415eb5 --- /dev/null +++ b/dbt/adapters/default/relation.py @@ -0,0 +1,214 @@ +from dbt.api import APIObject +from dbt.utils import filter_null_values + +PATH_SCHEMA = { + 'type': 'object', + 'properties': { + 'database': {'type': ['string', 'null']}, + 'schema': {'type': ['string', 'null']}, + 'identifier': {'type': 'string'}, + }, + 'required': ['database', 'schema', 'identifier'], +} + +POLICY_SCHEMA = { + 'type': 'object', + 'properties': { + 'database': {'type': 'boolean'}, + 'schema': {'type': 'boolean'}, + 'identifier': {'type': 'boolean'}, + }, + 'required': ['database', 'schema', 'identifier'], +} + + +class DefaultRelation(APIObject): + + Table = "table" + View = "view" + + RelationTypes = [ + Table, + View + ] + + DEFAULTS = { + 'metadata': { + '_type': 'DefaultRelation' + }, + 'quote_character': '"', + 'quote_policy': { + 'database': True, + 'schema': True, + 'identifier': True + }, + 'include_policy': { + 'database': False, + 'schema': True, + 'identifier': True + } + } + + SCHEMA = { + 'type': 'object', + 'properties': { + 'metadata': { + '_type': { + 'type': 'string', + 'const': 'DefaultRelation', + }, + }, + 'type': { + 'enum': RelationTypes + }, + 'path': PATH_SCHEMA, + 'include_policy': POLICY_SCHEMA, + 'quote_policy': POLICY_SCHEMA, + 'quote_character': {'type': 'string'}, + }, + 'required': ['metadata', 'type', 'path', 'include_policy', + 'quote_policy', 'quote_character'] + } + + PATH_ELEMENTS = ['database', 'schema', 'identifier'] + + def matches(self, database=None, schema=None, identifier=None): + search = filter_null_values({ + 'database': database, + 'schema': schema, + 'identifier': identifier + }) + + if not search: + # nothing was passed in + pass + + for k, v in search.items(): + if self.get_path_part(k) != v: + return False + + return True + + def get_path_part(self, part): + return self.path.get(part) + + def should_quote(self, part): + return self.quote_policy.get(part) + + def should_include(self, part): + return self.include_policy.get(part) + + def quote(self, database=None, schema=None, identifier=None): + policy = filter_null_values({ + 'database': database, + 'schema': schema, + 'identifier': identifier + }) + + return self.incorporate(quote_policy=policy) + + def include(self, database=None, schema=None, identifier=None): + policy = filter_null_values({ + 'database': database, + 'schema': schema, + 'identifier': identifier + }) + + return self.incorporate(include_policy=policy) + + def render(self): + parts = [] + + for k in ['database', 'schema', 'identifier']: + if self.should_include(k): + path_part = self.get_path_part(k) + + if path_part is None: + continue + + parts.append( + self.quote_if( + path_part, + self.should_quote(k))) + + if len(parts) == 0: + # TODO + raise RuntimeError( + "No path parts are included! Nothing to render.") + + return '.'.join(parts) + + def quote_if(self, identifier, should_quote): + if should_quote: + return self.quoted(identifier) + + return identifier + + def quoted(self, identifier): + return '{quote_char}{identifier}{quote_char}'.format( + quote_char=self.quote_character, + identifier=identifier) + + @classmethod + def create_from_node(cls, profile, node, **kwargs): + return cls.create_from_parts( + database=profile['dbname'], + schema=node['schema'], + identifier=node['name'], + **kwargs) + + @classmethod + def create_from_parts(cls, database=None, schema=None, + identifier=None, table_name=None, + type=None): + if table_name is None: + table_name = identifier + + return cls( + type=type, + path={ + 'database': database, + 'schema': schema, + 'identifier': identifier + }, + table_name=table_name) + + def __repr__(self): + return "<{} {}>".format(self.__class__.__name__, self.render()) + + def __str__(self): + return self.render() + + @property + def path(self): + return self.get('path', {}) + + @property + def database(self): + return self.path.get('database') + + @property + def schema(self): + return self.path.get('schema') + + @property + def identifier(self): + return self.path.get('identifier') + + # Here for compatibility with old Relation interface + @property + def name(self): + return self.identifier + + # Here for compatibility with old Relation interface + @property + def table(self): + return self._table_name + + @property + def is_table(self): + return self.type == self.Table + + @property + def is_view(self): + return self.type == self.View diff --git a/dbt/adapters/postgres/__init__.py b/dbt/adapters/postgres/__init__.py new file mode 100644 index 00000000000..82bb6ed5419 --- /dev/null +++ b/dbt/adapters/postgres/__init__.py @@ -0,0 +1,5 @@ +from dbt.adapters.postgres.impl import PostgresAdapter + +__all__ = [ + PostgresAdapter, +] diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres/impl.py similarity index 96% rename from dbt/adapters/postgres.py rename to dbt/adapters/postgres/impl.py index 0d6d8b2d5bb..0dca47a6009 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres/impl.py @@ -120,10 +120,10 @@ def list_relations(cls, profile, model_name=None): results = cursor.fetchall() - return [cls.Relation.create_from_parts(database=profile.get('dbname'), - schema=schema, - identifier=name, - type=type) + return [cls.Relation(database=profile.get('dbname'), + schema=schema, + identifier=name, + type=type) for (name, schema, type) in results] @classmethod diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift/__init__.py similarity index 100% rename from dbt/adapters/redshift.py rename to dbt/adapters/redshift/__init__.py diff --git a/dbt/adapters/relations/default.py b/dbt/adapters/relations/default.py deleted file mode 100644 index ba9e317d114..00000000000 --- a/dbt/adapters/relations/default.py +++ /dev/null @@ -1,200 +0,0 @@ -from dbt.utils import merge - - -# This should implement schema/name props for compatibility with old relation -# obj -class DefaultRelation(object): - QuoteCharacter = '"' - QuotePolicy = { - "database": True, - "schema": True, - "identifier": True - } - - IncludePolicy = { - "database": False, - "schema": True, - "identifier": True - } - - Table = "table" - View = "view" - - RelationTypes = [ - Table, - View - ] - - def matches(self, database=None, schema=None, identifier=None): - if database is not None and database != self.database: - return False - - if schema is not None and schema != self.schema: - return False - - if identifier is not None and identifier != self.identifier: - return False - - return True - - @property - def database(self): - return self._database - - @property - def type(self): - return self._type - - @property - def schema(self): - return self._schema - - @property - def identifier(self): - return self._identifier - - # Here for compatibility with old Relation interface - @property - def name(self): - return self.identifier - - # Here for compatibility with old Relation interface - @property - def table(self): - return self._table_name - - @property - def is_table(self): - return self.type == self.Table - - @property - def is_view(self): - return self.type == self.View - - def should_quote(self, part): - return self._quoting.get(part) - - def should_include(self, part): - return self._include.get(part) - - @property - def inclusion(self): - return self._include - - @property - def quoting(self): - return self._quoting - - def quote(self, database=None, schema=None, identifier=None): - raw_policy = { - "database": database, - "schema": schema, - "identifier": identifier - } - - policy_update = {k: v for (k,v) in raw_policy.items() if v is not None} - policy = self.quoting.copy() - policy.update(policy_update) - - return type(self)( - database=self.database, - schema=self.schema, - identifier=self.identifier, - quoting=policy, - include=self.inclusion) - - def include(self, database=None, schema=None, identifier=None): - raw_policy = { - "database": database, - "schema": schema, - "identifier": identifier - } - raw_policy = {k: v for (k, v) in raw_policy.items() if v is not None} - - policy = merge( - self.inclusion, - raw_policy) - - return type(self)( - database=self.database, - schema=self.schema, - identifier=self.identifier, - quoting=self.quoting, - include=policy) - - def render(self): - parts = [] - - if self.database is not None and self.should_include('database'): - parts.append(self.quote_if(self.database, self.should_quote('database'))) - - if self.schema is not None and self.should_include('schema'): - parts.append(self.quote_if(self.schema, self.should_quote('schema'))) - - if self.identifier is not None and self.should_include('identifier'): - parts.append(self.quote_if(self.identifier, self.should_quote('identifier'))) - - if len(parts) == 0: - # TODO - raise RuntimeError("Nothing to quote here....") - - return '.'.join(parts) - - def quote_if(self, identifier, should_quote): - if should_quote: - return self.quoted(identifier) - - return identifier - - @classmethod - def quoted(cls, s): - return '{quote_char}{identifier}{quote_char}'.format( - quote_char=cls.QuoteCharacter, identifier=s) - - @classmethod - def create_from_node(cls, profile, node, **kwargs): - return cls( - database=profile['dbname'], - schema=node['schema'], - identifier=node['name'], - **kwargs - ) - - @classmethod - def create_from_parts(cls, **kwargs): - # just use constructor - return cls(**kwargs) - - def __init__(self, database=None, schema=None, identifier=None, - table_name=None, type=None, quoting=None, include=None): - - self._database = database - self._schema = schema - self._identifier = identifier - self._type = type - - # This field is deprecated, but exists for backwards compatibility - # with the existing implementation of Relations - if table_name is None: - self._table_name = identifier - else: - self._table_name = table_name - - self._quoting = self.QuotePolicy.copy() - self._quoting.update(quoting or {}) - - self._include = self.IncludePolicy.copy() - self._include.update(include or {}) - - # validate - - def __repr__(self): - return "<{} {}>".format(self.__class__.__name__, self.render()) - - def __str__(self): - return self.render() - - -if __name__ == '__main__': - r = DefaultRelation("table", database='whatever', schema='ok', identifier='cool') - r.render() diff --git a/dbt/adapters/snowflake/__init__.py b/dbt/adapters/snowflake/__init__.py new file mode 100644 index 00000000000..b1ce3942300 --- /dev/null +++ b/dbt/adapters/snowflake/__init__.py @@ -0,0 +1,5 @@ +from dbt.adapters.snowflake.impl import SnowflakeAdapter + +__all__ = [ + SnowflakeAdapter, +] diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake/impl.py similarity index 100% rename from dbt/adapters/snowflake.py rename to dbt/adapters/snowflake/impl.py diff --git a/dbt/api/__init__.py b/dbt/api/__init__.py new file mode 100644 index 00000000000..c51e0a83548 --- /dev/null +++ b/dbt/api/__init__.py @@ -0,0 +1,5 @@ +from dbt.api.object import APIObject + +__all__ = [ + APIObject +] diff --git a/dbt/api/object.py b/dbt/api/object.py new file mode 100644 index 00000000000..d4c1089f0c9 --- /dev/null +++ b/dbt/api/object.py @@ -0,0 +1,82 @@ +import copy +from jsonschema import Draft4Validator + +from dbt.exceptions import ValidationException +from dbt.utils import deep_merge + + +class APIObject(dict): + """ + A serializable / deserializable object intended for + use in a future dbt API. + + To create a new object, you'll want to extend this + class, and then implement the SCHEMA property (a + valid JSON schema), the DEFAULTS property (default + settings for this object), and a static method that + calls this constructor. + """ + + SCHEMA = { + 'type': 'object', + 'properties': {} + } + + DEFAULTS = {} + + def __init__(self, *args, **kwargs): + """ + Create and validate an instance. Note that it's + not a good idea to override this. + """ + defaults = copy.deepcopy(self.DEFAULTS) + settings = copy.deepcopy(kwargs) + + d = deep_merge(defaults, settings) + super(APIObject, self).__init__(*args, **d) + self.__dict__ = self + self.validate() + + def incorporate(self, **kwargs): + """ + Given a list of kwargs, incorporate these arguments + into a new copy of this instance, and return the new + instance after validating. + """ + existing = copy.deepcopy(dict(self)) + updates = copy.deepcopy(kwargs) + return type(self)(**deep_merge(existing, updates)) + + def serialize(self): + """ + Return a dict representation of this object. + """ + return dict(self) + + @classmethod + def deserialize(cls, settings): + """ + Convert a dict representation of this object into + an actual object for internal use. + """ + return cls(**settings) + + def validate(self): + """ + Using the SCHEMA property, validate the attributes + of this instance. If any attributes are missing or + invalid, raise a ValidationException. + """ + validator = Draft4Validator(self.SCHEMA) + + errors = [] + + for error in validator.iter_errors(self.serialize()): + errors.append('property "{}", {}'.format( + ".".join(error.path), error.message)) + + if errors: + raise ValidationException( + 'Invalid arguments passed to "{}" instance: {}' + .format(type(self).__name__, + ", ".join(errors))) diff --git a/dbt/include/global_project/macros/materializations/bigquery.sql b/dbt/include/global_project/macros/materializations/bigquery.sql index bb12044ccba..179f3a78793 100644 --- a/dbt/include/global_project/macros/materializations/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/bigquery.sql @@ -2,11 +2,10 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = get_existing_relation_type(existing, identifier) -%} + {%- set existing = adapter.get_existing(dataset=schema, identifier=identifier) -%} - {%- if existing_type is not none -%} - {%- if existing_type == 'table' and not flags.FULL_REFRESH -%} + {%- if existing is not none -%} + {%- if existing.is_table and not flags.FULL_REFRESH -%} {# this is only intended for date partitioned tables, but we cant see that field in the context #} {% set error_message -%} Trying to create model '{{ identifier }}' as a view, but it already exists as a table. @@ -15,7 +14,7 @@ {{ exceptions.raise_compiler_error(error_message) }} {%- endif -%} - {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} + {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing.type)) }} {%- endif -%} -- build model @@ -57,8 +56,7 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = get_existing_relation_type(existing, identifier) -%} + {%- set existing = adapter.get_existing(dataset=schema, identifier=identifier) -%} {%- set verbose = config.get('verbose', False) -%} {%- set partitions = config.get('partitions') -%} @@ -76,13 +74,13 @@ Since dbt uses WRITE_TRUNCATE mode for tables, we only need to drop this thing if it is not a table. If it _is_ already a table, then we can overwrite it without downtime #} - {%- if existing_type is not none and existing_type != 'table' -%} - {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} + {%- if existing is not none and not existing.is_table -%} + {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing.type)) }} {%- endif -%} -- build model {% if partitions %} - {% set result = make_date_partitioned_table(model, partitions, (existing_type != 'table'), verbose) %} + {% set result = make_date_partitioned_table(model, partitions, (!existing.is_table), verbose) %} {% else %} {% set result = adapter.execute_model(model, 'table') %} {% endif %} diff --git a/dbt/utils.py b/dbt/utils.py index c423bb8335c..7b8a2ef1d07 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -388,3 +388,8 @@ def parse_cli_vars(var_string): logger.error( "The YAML provided in the --vars argument is not valid.\n") raise + + +def filter_null_values(input): + return dict((k, v) for (k, v) in input.items() + if v is not None) diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 00000000000..bf989686495 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,2 @@ +{name: test, profile: test, source-paths: [test/integration/010_permission_tests/models], + test-paths: [], version: '1.0'} diff --git a/setup.py b/setup.py index 5b62b0659a4..ed72c59d3ab 100644 --- a/setup.py +++ b/setup.py @@ -45,5 +45,6 @@ 'colorama==0.3.9', 'google-cloud-bigquery==0.29.0', 'agate>=1.6,<2', + 'jsonschema==2.6.0', ] ) From 57981e002b244e94bb9e8c247bdacf34aa0d4422 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 5 Apr 2018 15:48:34 -0400 Subject: [PATCH 14/61] rippin stuff out --- dbt/adapters/bigquery/impl.py | 4 ++-- dbt/adapters/default/impl.py | 8 +++++--- .../macros/materializations/bigquery.sql | 20 ++++++++++--------- dbt/node_runners.py | 18 ++++++++--------- dbt/runner.py | 8 +------- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index dc3194aadb2..03dcde624c7 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -239,8 +239,8 @@ def poll_until_job_completes(cls, job, timeout): raise job.exception() @classmethod - def make_date_partitioned_table(cls, profile, dataset_name, identifier, - model_name=None): + def make_date_partitioned_table(cls, profile, dataset_name, + identifier, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 200f763431f..1ae15510001 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -272,12 +272,14 @@ def list_relations(cls, profile, model_name=None): '`list_relations` is not implemented for this adapter!') @classmethod - def get_relation(cls, profile, **kwargs): - relations = cls.list_relations() + def get_relation(cls, profile, relations_list=None, + model_name=None, **kwargs): + if relations_list is None: + relations_list = cls.list_relations(profile, model_name) matches = [] - for relation in relations: + for relation in relations_list: if relation.matches(**kwargs): matches.append(relation) diff --git a/dbt/include/global_project/macros/materializations/bigquery.sql b/dbt/include/global_project/macros/materializations/bigquery.sql index 179f3a78793..97c8975d6e5 100644 --- a/dbt/include/global_project/macros/materializations/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/bigquery.sql @@ -2,19 +2,20 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.get_existing(dataset=schema, identifier=identifier) -%} + {%- set existing_relations = adapter.list_relations(dataset=schema) -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} - {%- if existing is not none -%} - {%- if existing.is_table and not flags.FULL_REFRESH -%} + {%- if old_relation is not none -%} + {%- if old_relation.is_table and not flags.FULL_REFRESH -%} {# this is only intended for date partitioned tables, but we cant see that field in the context #} {% set error_message -%} Trying to create model '{{ identifier }}' as a view, but it already exists as a table. - Either drop the '{{ schema }}.{{ identifier }}' table manually, or use --full-refresh + Either drop the {{ old_relation }} table manually, or use --full-refresh {%- endset %} {{ exceptions.raise_compiler_error(error_message) }} {%- endif -%} - {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing.type)) }} + {{ adapter.drop_relation(old_relation) }} {%- endif -%} -- build model @@ -56,7 +57,8 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.get_existing(dataset=schema, identifier=identifier) -%} + {%- set existing_relations = adapter.list_relations(dataset=schema) -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} {%- set verbose = config.get('verbose', False) -%} {%- set partitions = config.get('partitions') -%} @@ -74,13 +76,13 @@ Since dbt uses WRITE_TRUNCATE mode for tables, we only need to drop this thing if it is not a table. If it _is_ already a table, then we can overwrite it without downtime #} - {%- if existing is not none and not existing.is_table -%} - {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing.type)) }} + {%- if old_relation is not none and not old_relation.is_table -%} + {{ adapter.drop_relation(old_relation) }} {%- endif -%} -- build model {% if partitions %} - {% set result = make_date_partitioned_table(model, partitions, (!existing.is_table), verbose) %} + {% set result = make_date_partitioned_table(model, partitions, (!old_relation.is_table), verbose) %} {% else %} {% set result = adapter.execute_model(model, 'table') %} {% endif %} diff --git a/dbt/node_runners.py b/dbt/node_runners.py index bc74b74558e..e80cfcfe791 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -90,7 +90,7 @@ def is_ephemeral(cls, node): def is_ephemeral_model(cls, node): return cls.is_refable(node) and cls.is_ephemeral(node) - def safe_run(self, flat_graph, existing): + def safe_run(self, flat_graph): catchable_errors = (dbt.exceptions.CompilationException, dbt.exceptions.RuntimeException) @@ -105,7 +105,7 @@ def safe_run(self, flat_graph, existing): # for ephemeral nodes, we only want to compile, not run if not self.is_ephemeral_model(self.node): - result = self.run(compiled_node, existing, flat_graph) + result = self.run(compiled_node, flat_graph) except catchable_errors as e: if e.node is None: @@ -148,11 +148,11 @@ def safe_run(self, flat_graph, existing): def before_execute(self): raise NotImplementedException() - def execute(self, compiled_node, existing, flat_graph): + def execute(self, compiled_node, flat_graph): raise NotImplementedException() - def run(self, compiled_node, existing, flat_graph): - return self.execute(compiled_node, existing, flat_graph) + def run(self, compiled_node, flat_graph): + return self.execute(compiled_node, flat_graph) def after_execute(self, result): raise NotImplementedException() @@ -209,7 +209,7 @@ def before_execute(self): def after_execute(self, result): pass - def execute(self, compiled_node, existing, flat_graph): + def execute(self, compiled_node, flat_graph): return RunModelResult(compiled_node) def compile(self, flat_graph): @@ -401,7 +401,7 @@ def after_execute(self, result): track_model_run(self.node_index, self.num_nodes, result) self.print_result_line(result) - def execute(self, model, existing, flat_graph): + def execute(self, model, flat_graph): context = dbt.context.runtime.generate(model, self.project, flat_graph) materialization_macro = dbt.utils.get_materialization_macro( @@ -461,7 +461,7 @@ def execute_test(self, test): def before_execute(self): self.print_start_line() - def execute(self, test, existing, flat_graph): + def execute(self, test, flat_graph): status = self.execute_test(test) return RunModelResult(test, status=status) @@ -499,7 +499,7 @@ def before_execute(self): dbt.ui.printer.print_start_line(description, self.node_index, self.num_nodes) - def execute(self, compiled_node, existing_, flat_graph): + def execute(self, compiled_node, flat_graph): schema = compiled_node["schema"] table_name = compiled_node["name"] table = compiled_node["agate_table"] diff --git a/dbt/runner.py b/dbt/runner.py index efb9ff7427e..923bb9857e4 100644 --- a/dbt/runner.py +++ b/dbt/runner.py @@ -69,7 +69,6 @@ def get_runners(self, Runner, adapter, node_dependency_list): def call_runner(self, data): runner = data['runner'] - existing = data['existing'] flat_graph = data['flat_graph'] if runner.skip: @@ -79,7 +78,7 @@ def call_runner(self, data): if not runner.is_ephemeral_model(runner.node): runner.before_execute() - result = runner.safe_run(flat_graph, existing) + result = runner.safe_run(flat_graph) if not runner.is_ephemeral_model(runner.node): runner.after_execute(result) @@ -110,10 +109,6 @@ def execute_nodes(self, linker, Runner, flat_graph, node_dependency_list): dbt.ui.printer.print_timestamped_line("") schemas = list(Runner.get_model_schemas(flat_graph)) - if len(schemas) > 0: - existing = adapter.query_for_existing(profile, schemas) - else: - existing = {} node_runners = self.get_runners(Runner, adapter, node_dependency_list) pool = ThreadPool(num_threads) @@ -124,7 +119,6 @@ def execute_nodes(self, linker, Runner, flat_graph, node_dependency_list): args_list = [] for runner in runners: args_list.append({ - 'existing': existing, 'flat_graph': flat_graph, 'runner': runner }) From 9ecf92dfae42c3f62d4a2c5da904c68abdcd85b0 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 5 Apr 2018 17:09:12 -0400 Subject: [PATCH 15/61] tests passing(ish) --- TODO.md | 6 ++-- dbt/adapters/bigquery/__init__.py | 4 +-- dbt/adapters/bigquery/impl.py | 2 +- dbt/adapters/default/impl.py | 7 ++-- dbt/adapters/default/relation.py | 34 +++++++++++-------- dbt/adapters/postgres/impl.py | 9 ++--- dbt/adapters/relations/__init__.py | 0 dbt/adapters/relations/ephemeral.py | 8 ----- dbt/context/common.py | 1 + dbt/context/runtime.py | 19 ++++++----- .../macros/materializations/bigquery.sql | 2 +- .../macros/materializations/helpers.sql | 2 +- .../macros/materializations/incremental.sql | 19 ++++++----- .../macros/materializations/table.sql | 2 +- dbt/relation.py | 0 dbt/utils.py | 4 +++ requirements.txt | 1 + 17 files changed, 66 insertions(+), 54 deletions(-) delete mode 100644 dbt/adapters/relations/__init__.py delete mode 100644 dbt/adapters/relations/ephemeral.py delete mode 100644 dbt/relation.py diff --git a/TODO.md b/TODO.md index b5163ec0422..5a2be99eb7c 100644 --- a/TODO.md +++ b/TODO.md @@ -3,5 +3,7 @@ from the db (relation type, when we asked the db about it, columns, ??) - [ ] add get_relation, list_relations - fns / macros - - query_for_existing - - macro: get_existing_relation_type + - [ ] query_for_existing + - [ ] get_columns_in_table + - [ ] macro: get_existing_relation_type + - [ ] macro: create_table_as diff --git a/dbt/adapters/bigquery/__init__.py b/dbt/adapters/bigquery/__init__.py index 6ff9bcccd3f..3b0da501e22 100644 --- a/dbt/adapters/bigquery/__init__.py +++ b/dbt/adapters/bigquery/__init__.py @@ -1,5 +1,5 @@ -from dbt.adapters.bigquery.impl import BigqueryAdapter +from dbt.adapters.bigquery.impl import BigQueryAdapter __all__ = [ - BigqueryAdapter, + BigQueryAdapter, ] diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 03dcde624c7..fc95e7159fe 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -177,7 +177,7 @@ def list_relations(cls, profile, dataset, model_name=None): 'EXTERNAL': 'external' } - return [cls.Relation( + return [cls.Relation.create( project=credentials.get('project'), dataset=dataset, identifier=table.table_id, diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 1ae15510001..c0d2a45b996 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -149,9 +149,10 @@ def get_result_from_cursor(cls, cursor): def drop(cls, profile, schema, relation, relation_type, model_name=None): # add deprecation warning identifier = relation - relation = cls.Relation(schema=schema, - identifier=identifier, - type=relation_type) + relation = cls.Relation.create( + schema=schema, + identifier=identifier, + type=relation_type) return cls.drop_relation(profile, relation, model_name) diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 2ba0e415eb5..f43f3d37a8a 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -26,10 +26,12 @@ class DefaultRelation(APIObject): Table = "table" View = "view" + CTE = "cte" RelationTypes = [ Table, - View + View, + CTE ] DEFAULTS = { @@ -59,7 +61,7 @@ class DefaultRelation(APIObject): }, }, 'type': { - 'enum': RelationTypes + 'enum': RelationTypes + [None], }, 'path': PATH_SCHEMA, 'include_policy': POLICY_SCHEMA, @@ -151,27 +153,27 @@ def quoted(self, identifier): @classmethod def create_from_node(cls, profile, node, **kwargs): - return cls.create_from_parts( + return cls.create( database=profile['dbname'], schema=node['schema'], identifier=node['name'], **kwargs) @classmethod - def create_from_parts(cls, database=None, schema=None, - identifier=None, table_name=None, - type=None): + def create(cls, database=None, schema=None, + identifier=None, table_name=None, + type=None, **kwargs): if table_name is None: table_name = identifier - return cls( - type=type, - path={ - 'database': database, - 'schema': schema, - 'identifier': identifier - }, - table_name=table_name) + return cls(type=type, + path={ + 'database': database, + 'schema': schema, + 'identifier': identifier + }, + table_name=table_name, + **kwargs) def __repr__(self): return "<{} {}>".format(self.__class__.__name__, self.render()) @@ -209,6 +211,10 @@ def table(self): def is_table(self): return self.type == self.Table + @property + def is_cte(self): + return self.type == self.CTE + @property def is_view(self): return self.type == self.View diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 0dca47a6009..376fe9519af 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -120,10 +120,11 @@ def list_relations(cls, profile, model_name=None): results = cursor.fetchall() - return [cls.Relation(database=profile.get('dbname'), - schema=schema, - identifier=name, - type=type) + return [cls.Relation.create( + database=profile.get('dbname'), + schema=schema, + identifier=name, + type=type) for (name, schema, type) in results] @classmethod diff --git a/dbt/adapters/relations/__init__.py b/dbt/adapters/relations/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbt/adapters/relations/ephemeral.py b/dbt/adapters/relations/ephemeral.py deleted file mode 100644 index d2cedb9d0c1..00000000000 --- a/dbt/adapters/relations/ephemeral.py +++ /dev/null @@ -1,8 +0,0 @@ - - -from dbt.adapters.relations.default import DefaultRelation - - -class EphemeralRelation(DefaultRelation): - def render(self): - return '__dbt__CTE__{}'.format(self.identifier) diff --git a/dbt/context/common.py b/dbt/context/common.py index f048d44027a..4f9dae0c5fb 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -34,6 +34,7 @@ def __init__(self, model, adapter, profile): self.model = model self.adapter = adapter self.profile = profile + self.Relation = adapter.Relation # Fun with metaprogramming # Most adapter functions take `profile` as the first argument, and diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index 8a89de15246..5beb3ca4029 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -1,14 +1,10 @@ -import json - from dbt.adapters.factory import get_adapter -from dbt.compat import basestring +from dbt.node_types import NodeType +from dbt.utils import get_materialization, add_ephemeral_model_prefix import dbt.clients.jinja import dbt.context.common import dbt.flags -import dbt.utils - -from dbt.adapters.relations.ephemeral import EphemeralRelation from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -50,12 +46,17 @@ def do_ref(*args): target_model_name, target_model_package) - is_ephemeral = dbt.utils.get_materialization(target_model) == 'ephemeral' + is_ephemeral = (get_materialization(target_model) == 'ephemeral') + + adapter = get_adapter(profile) + if is_ephemeral: model['extra_ctes'][target_model_id] = None - return EphemeralRelation.create_from_node(profile, target_model) + return adapter.Relation.create( + type=adapter.Relation.CTE, + identifier=add_ephemeral_model_prefix( + target_model_name)).quote(identifier=False) else: - adapter = get_adapter(profile) return adapter.Relation.create_from_node(profile, target_model) return do_ref diff --git a/dbt/include/global_project/macros/materializations/bigquery.sql b/dbt/include/global_project/macros/materializations/bigquery.sql index 97c8975d6e5..77033f84f04 100644 --- a/dbt/include/global_project/macros/materializations/bigquery.sql +++ b/dbt/include/global_project/macros/materializations/bigquery.sql @@ -82,7 +82,7 @@ -- build model {% if partitions %} - {% set result = make_date_partitioned_table(model, partitions, (!old_relation.is_table), verbose) %} + {% set result = make_date_partitioned_table(model, partitions, (not old_relation.is_table), verbose) %} {% else %} {% set result = adapter.execute_model(model, 'table') %} {% endif %} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index cc7bb97168c..0691d66c628 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -49,6 +49,6 @@ {% macro drop_if_exists(existing, schema, name) %} {% set existing_type = get_existing_relation_type(existing, name) %} {% if existing_type is not none %} - {{ adapter.drop_relation(Relation(schema=schema, identifier=name, type=existing_type)) }} + {{ adapter.drop_relation(Relation.create(schema=schema, identifier=name, type=existing_type)) }} {% endif %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index 3b6991d0906..8a475e01d1c 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -19,23 +19,27 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = model['name'] + '__dbt_incremental_tmp' -%} + {%- set existing_relations = adapter.list_relations() -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} + {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier) -%} + {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = get_existing_relation_type(existing, identifier) -%} - {%- set exists_as_table = (existing_type == 'table') -%} + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and (full_refresh_mode or (existing_type not in (none, 'table')))) -%} + {%- set should_drop = (not should_truncate and full_refresh_mode or exists_not_as_table) -%} {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} -- setup - {% if existing_type is none -%} + {% if old_relation is none -%} -- noop {%- elif should_truncate -%} {{ adapter.truncate(schema, identifier) }} {%- elif should_drop -%} - {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} + {{ adapter.drop_relation(old_relation) }} {%- endif %} {{ run_hooks(pre_hooks, inside_transaction=False) }} @@ -44,8 +48,7 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model - {# TODO : Can we use `exists_as_table` here? Or did we need adapter.already_exists() #} - {% if force_create or not exists_as_table -%} + {% if force_create or old_relation is none or exists_not_as_table -%} {%- call statement('main') -%} {{ create_table_as(False, identifier, sql) }} {%- endcall -%} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 19938df2572..0f879358fd8 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -12,7 +12,7 @@ {% if existing_type == 'table' -%} {{ adapter.truncate(schema, identifier) }} {% elif existing_type == 'view' -%} - {{ adapter.drop_relation(Relation(schema=schema, identifier=identifier, type=existing_type)) }} + {{ adapter.drop_relation(Relation.create(schema=schema, identifier=identifier, type=existing_type)) }} {%- endif %} {%- endif %} diff --git a/dbt/relation.py b/dbt/relation.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbt/utils.py b/dbt/utils.py index 7b8a2ef1d07..d912f2134a2 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -393,3 +393,7 @@ def parse_cli_vars(var_string): def filter_null_values(input): return dict((k, v) for (k, v) in input.items() if v is not None) + + +def add_ephemeral_model_prefix(s): + return '__dbt__CTE__{}'.format(s) diff --git a/requirements.txt b/requirements.txt index 670aa77121b..78a23c40fdc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ snowflake-connector-python>=1.4.9 colorama==0.3.9 google-cloud-bigquery==0.29.0 agate>=1.6,<2 +jsonschema==2.6.0 From 370012d2b899e6c4b4f5a03243804a721a176faa Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 09:51:56 -0400 Subject: [PATCH 16/61] query_for_existing ripped out --- TODO.md | 3 +- dbt/adapters/default/impl.py | 5 +++ .../macros/adapters/postgres.sql | 9 ++++- .../macros/adapters/redshift.sql | 8 ++-- .../macros/materializations/archive.sql | 3 +- .../macros/materializations/helpers.sql | 6 +++ .../macros/materializations/incremental.sql | 20 ++++++---- .../macros/materializations/table.sql | 37 +++++++++++-------- .../macros/materializations/view.sql | 14 ++++--- 9 files changed, 68 insertions(+), 37 deletions(-) diff --git a/TODO.md b/TODO.md index 5a2be99eb7c..55562f5ea3b 100644 --- a/TODO.md +++ b/TODO.md @@ -3,7 +3,8 @@ from the db (relation type, when we asked the db about it, columns, ??) - [ ] add get_relation, list_relations - fns / macros - - [ ] query_for_existing + - [x] query_for_existing - [ ] get_columns_in_table + - [ ] rename - [ ] macro: get_existing_relation_type - [ ] macro: create_table_as diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index c0d2a45b996..057d94aa192 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -12,6 +12,7 @@ from dbt.contracts.connection import validate_connection from dbt.logger import GLOBAL_LOGGER as logger from dbt.schema import Column +from dbt.utils import filter_null_values from dbt.adapters.default.relation import DefaultRelation @@ -293,6 +294,10 @@ def get_relation(cls, profile, relations_list=None, return None + @classmethod + def reload_relation(cls, profile, relation, model_name=None): + return cls.get_relation(profile, model_name=model_name, + **filter_null_values(relation.path)) ### # SANE ANSI SQL DEFAULTS diff --git a/dbt/include/global_project/macros/adapters/postgres.sql b/dbt/include/global_project/macros/adapters/postgres.sql index 166c27e6b6a..bfa106422dd 100644 --- a/dbt/include/global_project/macros/adapters/postgres.sql +++ b/dbt/include/global_project/macros/adapters/postgres.sql @@ -1,6 +1,11 @@ -{% macro postgres__create_table_as(temporary, identifier, sql) -%} +{% macro postgres__create_table_as(temporary, relation, sql) -%} + {%- if temporary -%} + {%- set relation = relation.include(schema=False) -%} + {%- endif -%} + create {% if temporary: -%}temporary{%- endif %} table - {% if not temporary: -%}{{ schema }}.{%- endif %}{{ identifier }} as ( + {{ relation }} + as ( {{ sql }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/redshift.sql b/dbt/include/global_project/macros/adapters/redshift.sql index 9aae8a341f3..24567ce12f5 100644 --- a/dbt/include/global_project/macros/adapters/redshift.sql +++ b/dbt/include/global_project/macros/adapters/redshift.sql @@ -37,11 +37,9 @@ 'sort', validator=validation.any[list, basestring]) -%} - {% if temporary %} - {% set relation = identifier %} - {% else %} - {% set relation = schema ~ '.' ~ identifier %} - {% endif %} + {%- if temporary -%} + {%- set relation = relation.include(schema=False) -%} + {%- endif -%} create {% if temporary -%}temporary{%- endif %} table {{ relation }} {{ dist(_dist) }} diff --git a/dbt/include/global_project/macros/materializations/archive.sql b/dbt/include/global_project/macros/materializations/archive.sql index 658fb054284..9003b02b994 100644 --- a/dbt/include/global_project/macros/materializations/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive.sql @@ -110,6 +110,7 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = model['name'] + '__dbt_archival_tmp' -%} + {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, type='table') -%} {% call statement() %} {% set tmp_table_sql -%} @@ -121,7 +122,7 @@ {%- endset %} - {{ dbt.create_table_as(temporary=True, identifier=tmp_identifier, sql=tmp_table_sql) }} + {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} {% endcall %} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 0691d66c628..5f283c9ee22 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -52,3 +52,9 @@ {{ adapter.drop_relation(Relation.create(schema=schema, identifier=name, type=existing_type)) }} {% endif %} {% endmacro %} + +{% macro drop_relation_if_exists(relation) %} + {% if relation is not none %} + {{ adapter.drop_relation(relation) }} + {% endif %} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index 8a475e01d1c..7e211ea5c0f 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -20,8 +20,11 @@ {%- set tmp_identifier = model['name'] + '__dbt_incremental_tmp' -%} {%- set existing_relations = adapter.list_relations() -%} - {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} - {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier) -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + {%- set target_relation = adapter.Relation.create(identifier=identifier, schema=schema, type='table') -%} + {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, + schema=schema, type='table') -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} @@ -30,7 +33,7 @@ {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and full_refresh_mode or exists_not_as_table) -%} + {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%} {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} -- setup @@ -40,6 +43,7 @@ {{ adapter.truncate(schema, identifier) }} {%- elif should_drop -%} {{ adapter.drop_relation(old_relation) }} + {%- set old_relation = none -%} {%- endif %} {{ run_hooks(pre_hooks, inside_transaction=False) }} @@ -48,9 +52,9 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model - {% if force_create or old_relation is none or exists_not_as_table -%} + {% if force_create or old_relation is none -%} {%- call statement('main') -%} - {{ create_table_as(False, identifier, sql) }} + {{ create_table_as(False, target_relation, sql) }} {%- endcall -%} {%- else -%} {%- call statement() -%} @@ -64,7 +68,7 @@ or ({{ sql_where }}) is null {%- endset %} - {{ dbt.create_table_as(temporary=True, identifier=tmp_identifier, sql=tmp_table_sql) }} + {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} {%- endcall -%} @@ -82,10 +86,10 @@ {%- endif %} - insert into {{ schema }}.{{ identifier }} ({{ dest_cols_csv }}) + insert into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from {{ identifier }}__dbt_incremental_tmp + from {{ tmp_relation.include(schema=False) }} ); {% endcall %} {%- endif %} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 0f879358fd8..c2d742b95bb 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -2,17 +2,24 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = get_existing_relation_type(existing, identifier) -%} - {{ drop_if_exists(existing, schema, tmp_identifier) }} + {%- set existing_relations = adapter.list_relations() -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + {%- set target_relation = adapter.Relation.create(identifier=identifier, schema=schema, type='table') -%} + {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, + schema=schema, type='table') -%} - -- setup + -- drop the temp relation if it exists for some reason + {{ adapter.drop_relation(tmp_relation) }} + + -- setup: if the target relation already exists, truncate or drop it {% if non_destructive_mode -%} - {% if existing_type == 'table' -%} + {% if old_relation.is_table -%} {{ adapter.truncate(schema, identifier) }} - {% elif existing_type == 'view' -%} - {{ adapter.drop_relation(Relation.create(schema=schema, identifier=identifier, type=existing_type)) }} + {% elif old_relation.is_view -%} + {{ adapter.drop_relation(old_relation) }} + {%- set old_relation = none -%} {%- endif %} {%- endif %} @@ -24,21 +31,21 @@ -- build model {% call statement('main') -%} {%- if non_destructive_mode -%} - {%- if adapter.already_exists(schema, identifier) -%} - {{ create_table_as(True, tmp_identifier, sql) }} + {%- if old_relation is not none -%} + {{ create_table_as(True, tmp_relation, sql) }} {% set dest_columns = adapter.get_columns_in_table(schema, identifier) %} {% set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') %} - insert into {{ schema }}.{{ identifier }} ({{ dest_cols_csv }}) ( + insert into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from {{ tmp_identifier }} + from {{ tmp_relation }} ); {%- else -%} - {{ create_table_as(False, identifier, sql) }} + {{ create_table_as(False, target_relation, sql) }} {%- endif -%} {%- else -%} - {{ create_table_as(False, tmp_identifier, sql) }} + {{ create_table_as(False, tmp_relation, sql) }} {%- endif -%} {%- endcall %} @@ -48,8 +55,8 @@ {% if non_destructive_mode -%} -- noop {%- else -%} - {{ drop_if_exists(existing, schema, identifier) }} - {{ adapter.rename(schema, tmp_identifier, identifier) }} + {{ drop_relation_if_exists(target_relation) }} + {{ adapter.rename(schema, tmp_relation.identifier, target_relation.identifier) }} {%- endif %} -- `COMMIT` happens here diff --git a/dbt/include/global_project/macros/materializations/view.sql b/dbt/include/global_project/macros/materializations/view.sql index 6e172cebd9a..1b71e25ee89 100644 --- a/dbt/include/global_project/macros/materializations/view.sql +++ b/dbt/include/global_project/macros/materializations/view.sql @@ -3,14 +3,18 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = get_existing_relation_type(existing, identifier) -%} + + {%- set existing_relations = adapter.list_relations() -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} + {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, type='table') -%} + + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} {%- set has_transactional_hooks = (hooks | selectattr('transaction', 'equalto', True) | list | length) > 0 %} - {%- set should_ignore = non_destructive_mode and existing_type == 'view' %} + {%- set should_ignore = non_destructive_mode and exists_as_view %} {{ run_hooks(pre_hooks, inside_transaction=False) }} - {{ drop_if_exists(existing, schema, tmp_identifier) }} + {{ adapter.drop_relation(tmp_relation) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} @@ -37,7 +41,7 @@ -- cleanup {% if not should_ignore -%} - {{ drop_if_exists(existing, schema, identifier) }} + {{ drop_relation_if_exists(old_relation) }} {{ adapter.rename(schema, tmp_identifier, identifier) }} {%- endif %} From b2ee678e77f120cb741362ce82ab55a246739965 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 10:46:24 -0400 Subject: [PATCH 17/61] rename > rename_relation --- TODO.md | 1 + dbt/adapters/bigquery/impl.py | 6 +++++ dbt/adapters/default/impl.py | 16 +++++++++--- dbt/adapters/snowflake/impl.py | 10 +++---- .../macros/materializations/table.sql | 26 +++++++++++-------- .../macros/materializations/view.sql | 12 ++++++--- 6 files changed, 47 insertions(+), 24 deletions(-) diff --git a/TODO.md b/TODO.md index 55562f5ea3b..ddd2b8cf23a 100644 --- a/TODO.md +++ b/TODO.md @@ -8,3 +8,4 @@ - [ ] rename - [ ] macro: get_existing_relation_type - [ ] macro: create_table_as + - (go back through these after BQ pr is merged) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index fc95e7159fe..5fe4770007c 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -198,6 +198,12 @@ def rename(cls, profile, schema, from_name, to_name, model_name=None): raise dbt.exceptions.NotImplementedException( '`rename` is not implemented for this adapter!') + @classmethod + def rename_relation(cls, profile, from_relation, to_relation, + model_name=None): + raise dbt.exceptions.NotImplementedException( + '`rename_relation` is not implemented for this adapter!') + @classmethod def get_timeout(cls, conn): credentials = conn['credentials'] diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 057d94aa192..1f162baa598 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -43,7 +43,7 @@ class DefaultAdapter(object): "list_relations", "get_relation", "drop_relation", - + "rename_relation", "execute", "add_query", @@ -172,8 +172,18 @@ def truncate(cls, profile, schema, table, model_name=None): @classmethod def rename(cls, profile, schema, from_name, to_name, model_name=None): - from_relation = cls.render_relation(profile, schema, from_name) - sql = 'alter table {} rename to {}'.format(from_relation, to_name) + # add deprecation warning + return cls.rename_relation( + profile, + from_relation=cls.Relation(schema=schema, identifier=from_name), + to_relation=cls.Relation(identifier=to_name), + model_name=model_name) + + @classmethod + def rename_relation(cls, profile, from_relation, + to_relation, model_name=None): + sql = 'alter table {} rename to {}'.format( + from_relation, to_relation.include(schema=False)) connection, cursor = cls.add_query(profile, sql, model_name) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index a1d276f062b..18ead0035ce 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -137,12 +137,10 @@ def query_for_existing(cls, profile, schemas, model_name=None): return dict(existing) @classmethod - def rename(cls, profile, schema, from_name, to_name, model_name=None): - sql = (('alter table {schema}.{from_name} ' - 'rename to {schema}.{to_name}') - .format(schema=schema, - from_name=from_name, - to_name=to_name)) + def rename_relation(cls, profile, from_relation, + to_relation, model_name=None): + sql = 'alter table {} rename to {}'.format( + from_relation, to_relation) connection, cursor = cls.add_query(profile, sql, model_name) diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index c2d742b95bb..885dd5cb140 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -5,19 +5,23 @@ {%- set existing_relations = adapter.list_relations() -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, - schema=schema, identifier=identifier) -%} + schema=schema, identifier=identifier) -%} {%- set target_relation = adapter.Relation.create(identifier=identifier, schema=schema, type='table') -%} - {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, - schema=schema, type='table') -%} + {%- set intermediate_relation = adapter.Relation.create(identifier=tmp_identifier, + schema=schema, type='table') -%} + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + {%- set create_as_temporary = (exists_as_table and non_destructive_mode) -%} + -- drop the temp relation if it exists for some reason - {{ adapter.drop_relation(tmp_relation) }} + {{ adapter.drop_relation(intermediate_relation) }} -- setup: if the target relation already exists, truncate or drop it {% if non_destructive_mode -%} - {% if old_relation.is_table -%} + {% if exists_as_table -%} {{ adapter.truncate(schema, identifier) }} - {% elif old_relation.is_view -%} + {% elif exists_as_view -%} {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} {%- endif %} @@ -32,20 +36,20 @@ {% call statement('main') -%} {%- if non_destructive_mode -%} {%- if old_relation is not none -%} - {{ create_table_as(True, tmp_relation, sql) }} + {{ create_table_as(create_as_temporary, intermediate_relation, sql) }} {% set dest_columns = adapter.get_columns_in_table(schema, identifier) %} {% set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') %} insert into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from {{ tmp_relation }} + from {{ intermediate_relation }} ); {%- else -%} - {{ create_table_as(False, target_relation, sql) }} + {{ create_table_as(create_as_temporary, target_relation, sql) }} {%- endif -%} {%- else -%} - {{ create_table_as(False, tmp_relation, sql) }} + {{ create_table_as(create_as_temporary, intermediate_relation, sql) }} {%- endif -%} {%- endcall %} @@ -56,7 +60,7 @@ -- noop {%- else -%} {{ drop_relation_if_exists(target_relation) }} - {{ adapter.rename(schema, tmp_relation.identifier, target_relation.identifier) }} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} {%- endif %} -- `COMMIT` happens here diff --git a/dbt/include/global_project/macros/materializations/view.sql b/dbt/include/global_project/macros/materializations/view.sql index 1b71e25ee89..b9e1afe6973 100644 --- a/dbt/include/global_project/macros/materializations/view.sql +++ b/dbt/include/global_project/macros/materializations/view.sql @@ -5,8 +5,12 @@ {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing_relations = adapter.list_relations() -%} - {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} - {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, type='table') -%} + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + {%- set target_relation = adapter.Relation.create(identifier=identifier, schema=schema, + type='view') -%} + {%- set intermediate_relation = adapter.Relation.create(identifier=tmp_identifier, + schema=schema, type='view') -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} @@ -14,7 +18,7 @@ {%- set should_ignore = non_destructive_mode and exists_as_view %} {{ run_hooks(pre_hooks, inside_transaction=False) }} - {{ adapter.drop_relation(tmp_relation) }} + {{ adapter.drop_relation(intermediate_relation) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} @@ -42,7 +46,7 @@ -- cleanup {% if not should_ignore -%} {{ drop_relation_if_exists(old_relation) }} - {{ adapter.rename(schema, tmp_identifier, identifier) }} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} {%- endif %} {# From 803f1b7c17e8ae31a4d56bac6dc75c1647f686a2 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 17:52:00 -0400 Subject: [PATCH 18/61] bigquery, csv --- TODO.md | 7 +- dbt/adapters/bigquery/impl.py | 9 +- dbt/adapters/bigquery/relation.py | 77 ++++++++++++ dbt/adapters/default/impl.py | 17 ++- dbt/adapters/default/relation.py | 42 +++---- dbt/adapters/postgres/impl.py | 9 +- dbt/adapters/redshift/__init__.py | 110 +---------------- dbt/adapters/redshift/impl.py | 114 ++++++++++++++++++ .../macros/materializations/incremental.sql | 2 +- .../macros/materializations/table.sql | 2 +- 10 files changed, 250 insertions(+), 139 deletions(-) create mode 100644 dbt/adapters/bigquery/relation.py create mode 100644 dbt/adapters/redshift/impl.py diff --git a/TODO.md b/TODO.md index ddd2b8cf23a..a0f4ba127cf 100644 --- a/TODO.md +++ b/TODO.md @@ -5,7 +5,12 @@ - fns / macros - [x] query_for_existing - [ ] get_columns_in_table - - [ ] rename + - [x] rename + - [x] truncate + - [ ] drop + - [ ] reset_csv_table + - [ ] load_csv_rows - [ ] macro: get_existing_relation_type - [ ] macro: create_table_as - (go back through these after BQ pr is merged) +- [ ] deprecation warnings diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 5fe4770007c..c5e8512fe6f 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -9,6 +9,7 @@ import dbt.clients.agate_helper from dbt.adapters.postgres import PostgresAdapter +from dbt.adapters.bigquery.relation import BigQueryRelation from dbt.contracts.connection import validate_connection from dbt.logger import GLOBAL_LOGGER as logger @@ -33,6 +34,8 @@ class BigQueryAdapter(PostgresAdapter): "make_date_partitioned_table" ] + Relation = BigQueryRelation + SCOPE = ('https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/drive') @@ -482,8 +485,10 @@ def create_csv_table(cls, profile, schema, table_name, agate_table): @classmethod def reset_csv_table(cls, profile, schema, table_name, agate_table, - full_refresh=False): - cls.drop(profile, schema, table_name, "table") + full_refresh=False, model_name=None): + relation = cls.Relation(dataset=schema, identifier=table_name) + + cls.drop_relation(profile, relation, model_name) @classmethod def _agate_to_schema(cls, agate_table): diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py new file mode 100644 index 00000000000..b40b63bdb44 --- /dev/null +++ b/dbt/adapters/bigquery/relation.py @@ -0,0 +1,77 @@ +from dbt.adapters.default.relation import DefaultRelation +from dbt.utils import filter_null_values + + +class BigQueryRelation(DefaultRelation): + + DEFAULTS = { + 'metadata': { + '_type': 'DefaultRelation' + }, + 'quote_character': '"', + 'quote_policy': { + 'project': True, + 'dataset': True, + 'identifier': True + }, + 'include_policy': { + 'project': False, + 'dataset': True, + 'identifier': True + } + } + + PATH_DATASET = { + 'type': 'object', + 'properties': { + 'project': {'type': ['string', 'null']}, + 'dataset': {'type': ['string', 'null']}, + 'identifier': {'type': 'string'}, + }, + 'required': ['project', 'dataset', 'identifier'], + } + + POLICY_DATASET = { + 'type': 'object', + 'properties': { + 'project': {'type': 'boolean'}, + 'dataset': {'type': 'boolean'}, + 'identifier': {'type': 'boolean'}, + }, + 'required': ['project', 'dataset', 'identifier'], + } + + def matches(self, project=None, dataset=None, identifier=None): + search = filter_null_values({ + 'project': project, + 'dataset': dataset, + 'identifier': identifier + }) + + if not search: + # nothing was passed in + pass + + for k, v in search.items(): + if self.get_path_part(k) != v: + return False + + return True + + def quote(self, project=None, dataset=None, identifier=None): + policy = filter_null_values({ + 'project': project, + 'dataset': dataset, + 'identifier': identifier + }) + + return self.incorporate(quote_policy=policy) + + def include(self, project=None, dataset=None, identifier=None): + policy = filter_null_values({ + 'project': project, + 'dataset': dataset, + 'identifier': identifier + }) + + return self.incorporate(include_policy=policy) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 1f162baa598..27bebb3df38 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -165,7 +165,16 @@ def drop_relation(cls, profile, relation, model_name=None): @classmethod def truncate(cls, profile, schema, table, model_name=None): - relation = cls.render_relation(profile, schema, table) + # add deprecation warning + relation = cls.Relation.create( + schema=schema, + identifier=table, + type='table') + + return cls.truncate_relation(profile, relation, model_name) + + @classmethod + def truncate_relation(cls, profile, relation, model_name=None): sql = 'truncate table {}'.format(relation) connection, cursor = cls.add_query(profile, sql, model_name) @@ -175,8 +184,10 @@ def rename(cls, profile, schema, from_name, to_name, model_name=None): # add deprecation warning return cls.rename_relation( profile, - from_relation=cls.Relation(schema=schema, identifier=from_name), - to_relation=cls.Relation(identifier=to_name), + from_relation=cls.Relation.create( + schema=schema, identifier=from_name), + to_relation=cls.Relation.create( + identifier=to_name), model_name=model_name) @classmethod diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index f43f3d37a8a..207696e6354 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -1,26 +1,6 @@ from dbt.api import APIObject from dbt.utils import filter_null_values -PATH_SCHEMA = { - 'type': 'object', - 'properties': { - 'database': {'type': ['string', 'null']}, - 'schema': {'type': ['string', 'null']}, - 'identifier': {'type': 'string'}, - }, - 'required': ['database', 'schema', 'identifier'], -} - -POLICY_SCHEMA = { - 'type': 'object', - 'properties': { - 'database': {'type': 'boolean'}, - 'schema': {'type': 'boolean'}, - 'identifier': {'type': 'boolean'}, - }, - 'required': ['database', 'schema', 'identifier'], -} - class DefaultRelation(APIObject): @@ -51,6 +31,26 @@ class DefaultRelation(APIObject): } } + PATH_SCHEMA = { + 'type': 'object', + 'properties': { + 'database': {'type': ['string', 'null']}, + 'schema': {'type': ['string', 'null']}, + 'identifier': {'type': 'string'}, + }, + 'required': ['database', 'schema', 'identifier'], + } + + POLICY_SCHEMA = { + 'type': 'object', + 'properties': { + 'database': {'type': 'boolean'}, + 'schema': {'type': 'boolean'}, + 'identifier': {'type': 'boolean'}, + }, + 'required': ['database', 'schema', 'identifier'], + } + SCHEMA = { 'type': 'object', 'properties': { @@ -72,8 +72,6 @@ class DefaultRelation(APIObject): 'quote_policy', 'quote_character'] } - PATH_ELEMENTS = ['database', 'schema', 'identifier'] - def matches(self, database=None, schema=None, identifier=None): search = filter_null_values({ 'database': database, diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 376fe9519af..1cbc7321df4 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -201,12 +201,15 @@ def create_csv_table(cls, profile, schema, table_name, agate_table): @classmethod def reset_csv_table(cls, profile, schema, table_name, agate_table, - full_refresh=False): + full_refresh=False, model_name=None): + relation = cls.Relation.create( + schema=schema, identifier=table_name, type='table') + if full_refresh: - cls.drop(profile, schema, table_name, 'table', None) + cls.drop_relation(profile, relation, model_name) cls.create_csv_table(profile, schema, table_name, agate_table) else: - cls.truncate(profile, schema, table_name) + cls.truncate_relation(profile, relation, model_name) @classmethod def load_csv_rows(cls, profile, schema, table_name, agate_table): diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index a8ecb661657..a92a6acbd80 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -1,107 +1,5 @@ -import multiprocessing +from dbt.adapters.redshift.impl import RedshiftAdapter -from dbt.adapters.postgres import PostgresAdapter -from dbt.logger import GLOBAL_LOGGER as logger # noqa - - -drop_lock = multiprocessing.Lock() - - -class RedshiftAdapter(PostgresAdapter): - - @classmethod - def type(cls): - return 'redshift' - - @classmethod - def date_function(cls): - return 'getdate()' - - @classmethod - def _get_columns_in_table_sql(cls, schema_name, table_name): - # TODO : how do we make this a macro? - if schema_name is None: - table_schema_filter = '1=1' - else: - table_schema_filter = "table_schema = '{schema_name}'".format( - schema_name=schema_name) - - sql = """ - with bound_views as ( - select - table_schema, - column_name, - data_type, - character_maximum_length - - from information_schema.columns - where table_name = '{table_name}' - ), - - unbound_views as ( - select - view_schema, - col_name, - col_type, - case - when col_type like 'character%' - then nullif(REGEXP_SUBSTR(col_type, '[0-9]+'), '')::int - else null - end as character_maximum_length - - from pg_get_late_binding_view_cols() - cols(view_schema name, view_name name, col_name name, - col_type varchar, col_num int) - where view_name = '{table_name}' - ), - - unioned as ( - select * from bound_views - union all - select * from unbound_views - ) - - select column_name, data_type, character_maximum_length - from unioned - where {table_schema_filter} - """.format(table_name=table_name, - table_schema_filter=table_schema_filter).strip() - return sql - - @classmethod - def drop(cls, profile, relation, model_name=None): - global drop_lock - - to_return = None - - try: - drop_lock.acquire() - - connection = cls.get_connection(profile, model_name) - - if connection.get('transaction_open'): - cls.commit(profile, connection) - - cls.begin(profile, connection.get('name')) - - to_return = super(PostgresAdapter, cls).drop( - profile, 'table', relation, model_name) - - cls.commit(profile, connection) - cls.begin(profile, connection.get('name')) - - return to_return - - finally: - drop_lock.release() - - @classmethod - def convert_text_type(cls, agate_table, col_idx): - column = agate_table.columns[col_idx] - lens = (len(d.encode("utf-8")) for d in column.values_without_nulls()) - max_len = max(lens) if lens else 64 - return "varchar({})".format(max_len) - - @classmethod - def convert_time_type(cls, agate_table, col_idx): - return "varchar(24)" +__all__ = [ + RedshiftAdapter, +] diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py new file mode 100644 index 00000000000..59344f02282 --- /dev/null +++ b/dbt/adapters/redshift/impl.py @@ -0,0 +1,114 @@ +import multiprocessing + +from dbt.adapters.postgres import PostgresAdapter +from dbt.logger import GLOBAL_LOGGER as logger # noqa + + +drop_lock = multiprocessing.Lock() + + +class RedshiftAdapter(PostgresAdapter): + + @classmethod + def type(cls): + return 'redshift' + + @classmethod + def date_function(cls): + return 'getdate()' + + @classmethod + def _get_columns_in_table_sql(cls, schema_name, table_name): + # TODO : how do we make this a macro? + if schema_name is None: + table_schema_filter = '1=1' + else: + table_schema_filter = "table_schema = '{schema_name}'".format( + schema_name=schema_name) + + sql = """ + with bound_views as ( + select + table_schema, + column_name, + data_type, + character_maximum_length + + from information_schema.columns + where table_name = '{table_name}' + ), + + unbound_views as ( + select + view_schema, + col_name, + col_type, + case + when col_type like 'character%' + then nullif(REGEXP_SUBSTR(col_type, '[0-9]+'), '')::int + else null + end as character_maximum_length + + from pg_get_late_binding_view_cols() + cols(view_schema name, view_name name, col_name name, + col_type varchar, col_num int) + where view_name = '{table_name}' + ), + + unioned as ( + select * from bound_views + union all + select * from unbound_views + ) + + select column_name, data_type, character_maximum_length + from unioned + where {table_schema_filter} + """.format(table_name=table_name, + table_schema_filter=table_schema_filter).strip() + return sql + + @classmethod + def drop_relation(cls, profile, relation, model_name=None): + """ + In Redshift, DROP TABLE cannot be used inside a transaction. + This creates issues with multiple threads, so we need to + lock around calls to the underlying drop_relation() function. + + https://docs.aws.amazon.com/redshift/latest/dg/r_DROP_TABLE.html + """ + global drop_lock + + to_return = None + + try: + drop_lock.acquire() + + connection = cls.get_connection(profile, model_name) + + if connection.get('transaction_open'): + cls.commit(profile, connection) + + cls.begin(profile, connection.get('name')) + + to_return = super(PostgresAdapter, cls).drop_relation( + profile, relation, model_name) + + cls.commit(profile, connection) + cls.begin(profile, connection.get('name')) + + return to_return + + finally: + drop_lock.release() + + @classmethod + def convert_text_type(cls, agate_table, col_idx): + column = agate_table.columns[col_idx] + lens = (len(d.encode("utf-8")) for d in column.values_without_nulls()) + max_len = max(lens) if lens else 64 + return "varchar({})".format(max_len) + + @classmethod + def convert_time_type(cls, agate_table, col_idx): + return "varchar(24)" diff --git a/dbt/include/global_project/macros/materializations/incremental.sql b/dbt/include/global_project/macros/materializations/incremental.sql index 7e211ea5c0f..200411eec65 100644 --- a/dbt/include/global_project/macros/materializations/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental.sql @@ -40,7 +40,7 @@ {% if old_relation is none -%} -- noop {%- elif should_truncate -%} - {{ adapter.truncate(schema, identifier) }} + {{ adapter.truncate_relation(old_relation) }} {%- elif should_drop -%} {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} diff --git a/dbt/include/global_project/macros/materializations/table.sql b/dbt/include/global_project/macros/materializations/table.sql index 885dd5cb140..ad89e513114 100644 --- a/dbt/include/global_project/macros/materializations/table.sql +++ b/dbt/include/global_project/macros/materializations/table.sql @@ -20,7 +20,7 @@ -- setup: if the target relation already exists, truncate or drop it {% if non_destructive_mode -%} {% if exists_as_table -%} - {{ adapter.truncate(schema, identifier) }} + {{ adapter.truncate_relation(old_relation) }} {% elif exists_as_view -%} {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} From f253261d07641b40674a617d23b18e26dd4fde52 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 21:47:12 -0400 Subject: [PATCH 19/61] remove extraneous dbt_project file --- dbt_project.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 dbt_project.yml diff --git a/dbt_project.yml b/dbt_project.yml deleted file mode 100644 index bf989686495..00000000000 --- a/dbt_project.yml +++ /dev/null @@ -1,2 +0,0 @@ -{name: test, profile: test, source-paths: [test/integration/010_permission_tests/models], - test-paths: [], version: '1.0'} From b9c7d924c1d525b1f9defb245e2e519f83ab9824 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 22:24:32 -0400 Subject: [PATCH 20/61] move types into "api" namespace, fix archival --- dbt/adapters/bigquery/relation.py | 6 ++-- dbt/adapters/default/relation.py | 4 ++- dbt/adapters/redshift/impl.py | 13 +++++-- dbt/exceptions.py | 13 +++++-- .../materializations/archive/archive.sql | 34 +++++++++++++++---- .../macros/materializations/helpers.sql | 2 +- .../incremental/incremental.sql | 4 +-- .../materializations/table/bigquery_table.sql | 2 +- .../macros/materializations/table/table.sql | 4 +-- .../macros/materializations/view/view.sql | 4 +-- 10 files changed, 64 insertions(+), 22 deletions(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index b40b63bdb44..fd34509ed44 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -21,7 +21,7 @@ class BigQueryRelation(DefaultRelation): } } - PATH_DATASET = { + PATH_SCHEMA = { 'type': 'object', 'properties': { 'project': {'type': ['string', 'null']}, @@ -31,7 +31,7 @@ class BigQueryRelation(DefaultRelation): 'required': ['project', 'dataset', 'identifier'], } - POLICY_DATASET = { + POLICY_SCHEMA = { 'type': 'object', 'properties': { 'project': {'type': 'boolean'}, @@ -41,6 +41,8 @@ class BigQueryRelation(DefaultRelation): 'required': ['project', 'dataset', 'identifier'], } + PATH_ELEMENTS = ['project', 'dataset', 'identifier'] + def matches(self, project=None, dataset=None, identifier=None): search = filter_null_values({ 'project': project, diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 207696e6354..55cb4c83202 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -72,6 +72,8 @@ class DefaultRelation(APIObject): 'quote_policy', 'quote_character'] } + PATH_ELEMENTS = ['database', 'schema', 'identifier'] + def matches(self, database=None, schema=None, identifier=None): search = filter_null_values({ 'database': database, @@ -119,7 +121,7 @@ def include(self, database=None, schema=None, identifier=None): def render(self): parts = [] - for k in ['database', 'schema', 'identifier']: + for k in self.PATH_ELEMENTS: if self.should_include(k): path_part = self.get_path_part(k) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 6dd823f4708..310632773fc 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -93,9 +93,16 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): @classmethod def drop_relation(cls, profile, relation, model_name=None): """ - In Redshift, DROP TABLE cannot be used inside a transaction. - This creates issues with multiple threads, so we need to - lock around calls to the underlying drop_relation() function. + In Redshift, DROP TABLE ... CASCADE should not be used + inside a transaction. Redshift doesn't prevent the CASCADE + part from conflicting with concurrent transactions. If we do + attempt to drop two tables with CASCADE at once, we'll often + get the dreaded: + + table was dropped by a concurrent transaction + + So, we need to lock around calls to the underlying + drop_relation() function. https://docs.aws.amazon.com/redshift/latest/dg/r_DROP_TABLE.html """ diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 94d72b7f497..d6eb4eeeb4a 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -248,9 +248,18 @@ def missing_config(model, name): model) -def missing_relation(relation_name, model=None): +def missing_relation(relation, model=None): raise_compiler_error( - "Relation {} not found!".format(relation_name), + "Relation {} not found!".format(relation), + model) + + +def relation_wrong_type(relation, expected_type, model=None): + raise_compiler_error( + ('Relation {} exists, but it is a {}, not a {}! ' + 'You can resolve this by renaming the relation, or ' + 'dropping the existing relation.') + .format(str(relation), relation.type, expected_type), model) diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/dbt/include/global_project/macros/materializations/archive/archive.sql index 9003b02b994..c1977bf3f8b 100644 --- a/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -72,16 +72,37 @@ {% materialization archive, default %} {%- set config = model['config'] -%} + {%- set existing_relations = adapter.list_relations() -%} + + {%- set target_schema = config.get('target_schema') -%} + {%- set target_table = config.get('target_table') -%} + + {%- set source_relation = adapter.get_relation( + relations_list=existing_relations, + schema=config.get('source_schema'), + identifier=config.get('source_table')) -%} + + {%- set target_relation = adapter.get_relation( + relations_list=existing_relations, + schema=target_schema, + identifier=target_table) -%} + {%- set source_schema = config.get('source_schema') -%} {%- set source_table = config.get('source_table') -%} - {%- if not adapter.already_exists(source_schema, source_table) -%} - {{ exceptions.missing_relation(source_table) }} + {%- if source_relation is none -%} + {{ exceptions.missing_relation(source_relation) }} + {%- endif -%} + + {%- if target_relation is none -%} + {%- set target_relation = api.Relation.create( + schema=target_schema, + identifier=target_table) -%} + {%- elif not target_relation.is_table -%} + {{ exceptions.relation_wrong_type(target_relation, 'table') }} {%- endif -%} {%- set source_columns = adapter.get_columns_in_table(source_schema, source_table) -%} - {%- set target_schema = config.get('target_schema') -%} - {%- set target_table = config.get('target_table') -%} {%- set unique_key = config.get('unique_key') -%} {%- set updated_at = config.get('updated_at') -%} {%- set dest_columns = source_columns + [ @@ -104,13 +125,14 @@ {% for col in missing_columns %} {% call statement() %} - alter table {{ target_schema }}.{{ target_table }} add column "{{ col.name }}" {{ col.data_type }}; + alter table {{ target_relation }} + add column "{{ col.name }}" {{ col.data_type }}; {% endcall %} {% endfor %} {%- set identifier = model['name'] -%} {%- set tmp_identifier = model['name'] + '__dbt_archival_tmp' -%} - {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, type='table') -%} + {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='table') -%} {% call statement() %} {% set tmp_table_sql -%} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 5f283c9ee22..33256fe0d4a 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -49,7 +49,7 @@ {% macro drop_if_exists(existing, schema, name) %} {% set existing_type = get_existing_relation_type(existing, name) %} {% if existing_type is not none %} - {{ adapter.drop_relation(Relation.create(schema=schema, identifier=name, type=existing_type)) }} + {{ adapter.drop_relation(api.Relation.create(schema=schema, identifier=name, type=existing_type)) }} {% endif %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql index 200411eec65..582b162ccd3 100644 --- a/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -22,8 +22,8 @@ {%- set existing_relations = adapter.list_relations() -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} - {%- set target_relation = adapter.Relation.create(identifier=identifier, schema=schema, type='table') -%} - {%- set tmp_relation = adapter.Relation.create(identifier=tmp_identifier, + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} + {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, schema=schema, type='table') -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql index 0a0635bb4a4..9885a6990d5 100644 --- a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql +++ b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql @@ -60,7 +60,7 @@ -- build model {% if partitions %} - {{ set result = make_date_partitioned_table(model, partitions, (not old_relation.is_table), verbose) }} + {{ make_date_partitioned_table(model, partitions, (not old_relation.is_table), verbose) }} {% else %} {% call statement('main') -%} {{ create_table_as(False, identifier, sql) }} diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql index ad89e513114..e8da50425fe 100644 --- a/dbt/include/global_project/macros/materializations/table/table.sql +++ b/dbt/include/global_project/macros/materializations/table/table.sql @@ -6,8 +6,8 @@ {%- set existing_relations = adapter.list_relations() -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} - {%- set target_relation = adapter.Relation.create(identifier=identifier, schema=schema, type='table') -%} - {%- set intermediate_relation = adapter.Relation.create(identifier=tmp_identifier, + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} + {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, schema=schema, type='table') -%} {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} diff --git a/dbt/include/global_project/macros/materializations/view/view.sql b/dbt/include/global_project/macros/materializations/view/view.sql index b9e1afe6973..f49fc8017e2 100644 --- a/dbt/include/global_project/macros/materializations/view/view.sql +++ b/dbt/include/global_project/macros/materializations/view/view.sql @@ -7,9 +7,9 @@ {%- set existing_relations = adapter.list_relations() -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} - {%- set target_relation = adapter.Relation.create(identifier=identifier, schema=schema, + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='view') -%} - {%- set intermediate_relation = adapter.Relation.create(identifier=tmp_identifier, + {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, schema=schema, type='view') -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} From 29f93b9bafc553f93e13fa8a8ea2ae16b49e06a0 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 22:58:27 -0400 Subject: [PATCH 21/61] rip out table_exists --- TODO.md | 1 + dbt/adapters/bigquery/impl.py | 6 --- dbt/adapters/default/impl.py | 35 +++++-------- dbt/adapters/snowflake/impl.py | 12 ++--- dbt/adapters/snowflake/relation.py | 50 +++++++++++++++++++ dbt/deprecations.py | 10 +++- dbt/exceptions.py | 11 ++-- .../materializations/view/bigquery_view.sql | 35 ++++++++----- dbt/node_runners.py | 6 +-- 9 files changed, 107 insertions(+), 59 deletions(-) create mode 100644 dbt/adapters/snowflake/relation.py diff --git a/TODO.md b/TODO.md index a0f4ba127cf..67424a89879 100644 --- a/TODO.md +++ b/TODO.md @@ -10,6 +10,7 @@ - [ ] drop - [ ] reset_csv_table - [ ] load_csv_rows + - [ ] handle_csv_table - [ ] macro: get_existing_relation_type - [ ] macro: create_table_as - (go back through these after BQ pr is merged) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 915323a9be5..d57af9d14b9 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -194,12 +194,6 @@ def list_relations(cls, profile, dataset, model_name=None): type=relation_types.get(table.table_type)) for table in all_tables] - @classmethod - def table_exists(cls, profile, schema, table, model_name=None): - tables = cls.query_for_existing(profile, schema, model_name) - exists = tables.get(table) is not None - return exists - @classmethod def drop_relation(cls, profile, relation, model_name=None): conn = cls.get_connection(profile, model_name) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index e6d2c0e4835..4460faebbc1 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -691,22 +691,11 @@ def drop_schema(cls, profile, schema, model_name=None): sql = cls.get_drop_schema_sql(schema) return cls.add_query(profile, sql, model_name) - @classmethod - def table_existing_type(cls, profile, schema, table, model_name=None): - tables = cls.query_for_existing(profile, schema, model_name) - return tables.get(table) - - @classmethod - def table_exists(cls, profile, schema, table, model_name=None): - rel_type = cls.table_existing_type(profile, schema, table, model_name) - return rel_type is not None - @classmethod def already_exists(cls, profile, schema, table, model_name=None): - """ - Alias for `table_exists`. - """ - return cls.table_exists(profile, schema, table, model_name) + # TODO add deprecation warning + relation = cls.get_relation(schema=schema, identifier=table) + return relation is not None @classmethod def quote(cls, identifier): @@ -724,17 +713,19 @@ def render_relation(cls, profile, schema, name): @classmethod def handle_csv_table(cls, profile, schema, table_name, agate_table, full_refresh=False): - existing = cls.query_for_existing(profile, schema) - upcased_existing = {k.upper(): v for k, v in existing.items()} - existing_type = upcased_existing.get(table_name.upper()) - if existing_type and existing_type != "table": - raise dbt.exceptions.RuntimeException( - "Cannot seed to '{}', it is a view".format(table_name)) - if existing_type: + relation = cls.get_relation( + profile, schema=schema, identifier=table_name) + + if relation is None: + cls.create_csv_table(profile, schema, table_name, agate_table) + + elif relation.is_table or full_refresh: cls.reset_csv_table(profile, schema, table_name, agate_table, full_refresh=full_refresh) + else: - cls.create_csv_table(profile, schema, table_name, agate_table) + dbt.exceptions.relation_wrong_type(relation, 'table') + cls.load_csv_rows(profile, schema, table_name, agate_table) cls.commit_if_has_connection(profile, None) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 18ead0035ce..708545d232c 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -9,15 +9,16 @@ import dbt.compat import dbt.exceptions -import dbt.flags as flags from dbt.adapters.postgres import PostgresAdapter -from dbt.contracts.connection import validate_connection +from dbt.adapters.snowflake.relation import SnowflakeRelation from dbt.logger import GLOBAL_LOGGER as logger class SnowflakeAdapter(PostgresAdapter): + Relation = SnowflakeRelation + @classmethod @contextmanager def exception_handler(cls, profile, sql, model_name=None, @@ -102,13 +103,6 @@ def open_connection(cls, connection): return result - @classmethod - def table_existing_type(cls, profile, schema, table, model_name=None): - # this is a case-insensitive lookup - relations = cls.query_for_existing(profile, schema, model_name) - upcased_relations = {k.upper(): v for (k, v) in relations.items()} - return upcased_relations.get(table.upper()) - @classmethod def query_for_existing(cls, profile, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py new file mode 100644 index 00000000000..00aa9471e8b --- /dev/null +++ b/dbt/adapters/snowflake/relation.py @@ -0,0 +1,50 @@ +from dbt.adapters.default.relation import DefaultRelation +from dbt.utils import filter_null_values + + +class SnowflakeRelation(DefaultRelation): + DEFAULTS = { + 'metadata': { + '_type': 'DefaultRelation' + }, + 'quote_character': '"', + 'quote_policy': { + 'database': False, + 'schema': False, + 'identifier': False, + }, + 'include_policy': { + 'database': False, + 'schema': True, + 'identifier': True, + } + } + + def matches(self, database=None, schema=None, identifier=None): + search = filter_null_values({ + 'database': database, + 'schema': schema, + 'identifier': identifier + }) + + if not search: + # nothing was passed in + pass + + for k, v in search.items(): + # snowflake upcases unquoted identiifers. so, when + # comparing unquoted identifiers, use case insensitive + # matching. when comparing quoted identifiers, use case + # sensitive matching. + if self.should_quote(k): + if self.get_path_part(k) != v: + return False + + else: + if self.get_path_part(k) == v.upper(): + return False + + return True + + def get_path_part(self, part): + return self.path.get(part) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index aa0e36b3309..72ac37597af 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -34,8 +34,13 @@ class SeedDropExistingDeprecation(DBTDeprecation): class OldStyleAdapterFunctionsDeprecations(DBTDeprecation): + # TODO add link to docs name = 'old-style-adapter-functions' - description = """Use the new version of this thing?""" + description = """ + This is an old-style adapter function (takes a schema and identifier + instead of a Relation object). Please use the new-style functions. + You can find more information in the documentation. (ADD LINK) + """ def warn(name, *args, **kwargs): @@ -55,7 +60,8 @@ def warn(name, *args, **kwargs): deprecations_list = [ DBTRepositoriesDeprecation(), - SeedDropExistingDeprecation() + SeedDropExistingDeprecation(), + OldStyleAdapterFunctionsDeprecations(), ] deprecations = {d.name: d for d in deprecations_list} diff --git a/dbt/exceptions.py b/dbt/exceptions.py index d6eb4eeeb4a..fd11fc80f73 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -256,10 +256,13 @@ def missing_relation(relation, model=None): def relation_wrong_type(relation, expected_type, model=None): raise_compiler_error( - ('Relation {} exists, but it is a {}, not a {}! ' - 'You can resolve this by renaming the relation, or ' - 'dropping the existing relation.') - .format(str(relation), relation.type, expected_type), + ('Trying to create {intended_type} {relation}, ' + 'but it currently exists as a {current_type}. Either ' + 'drop {relation} manually, or run dbt with ' + '`--full-refresh` and dbt will drop it for you.') + .format(relation=relation, + current_type=relation.type, + expected_type=expected_type), model) diff --git a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql index 4b21ee5e44f..b3cd317c6bc 100644 --- a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql +++ b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql @@ -3,24 +3,33 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} - - {%- if existing_type is not none -%} - {%- if existing_type == 'table' and not flags.FULL_REFRESH -%} - {# this is only intended for date partitioned tables, but we cant see that field in the context #} - {% set error_message -%} - Trying to create model '{{ identifier }}' as a view, but it already exists as a table. - Either drop the '{{ schema }}.{{ identifier }}' table manually, or use --full-refresh - {%- endset %} - {{ exceptions.raise_compiler_error(error_message) }} + + {%- set existing_relations = adapter.list_relations() -%} + + {%- set old_relation = adapter.get_relation( + relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, + type='view') -%} + + {%- set intermediate_relation = api.Relation.create( + identifier=tmp_identifier, + schema=schema, type='view') -%} + + + -- drop if exists + {%- if old_relation is not none -%} + {%- if old_relation.is_table and not flags.FULL_REFRESH -%} + {{ exceptions.relation_wrong_type(old_relation, 'view') }} {%- endif -%} - {{ adapter.drop(schema, identifier, existing_type) }} + {{ adapter.drop_relation(old_relation) }} {%- endif -%} -- build model - {% if existing_type == 'view' and non_destructive_mode -%} + {% if old_relation is not none and old_relation.is_view and non_destructive_mode -%} {% call noop_statement('main', status="PASS", res=None) -%} -- Not running : non-destructive mode {{ sql }} diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 6004b76b0cb..d7da9c5fa1d 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -259,8 +259,8 @@ def call_get_missing_columns(from_schema, from_table, profile, from_schema, from_table, to_schema, to_table, node.get('name')) - def call_table_exists(schema, table): - return adapter.table_exists( + def call_already_exists(schema, table): + return adapter.already_exists( profile, schema, table, node.get('name')) return { @@ -268,7 +268,7 @@ def call_table_exists(schema, table): "invocation_id": dbt.tracking.active_user.invocation_id, "get_columns_in_table": call_get_columns_in_table, "get_missing_columns": call_get_missing_columns, - "already_exists": call_table_exists, + "already_exists": call_already_exists, } @classmethod From 01c7cff829c36b402068d5ea8b5524b685ad7271 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 22:59:28 -0400 Subject: [PATCH 22/61] include relations --- dbt/adapters/bigquery/__init__.py | 2 ++ dbt/adapters/snowflake/__init__.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/dbt/adapters/bigquery/__init__.py b/dbt/adapters/bigquery/__init__.py index 3b0da501e22..f5c01e93ffc 100644 --- a/dbt/adapters/bigquery/__init__.py +++ b/dbt/adapters/bigquery/__init__.py @@ -1,5 +1,7 @@ from dbt.adapters.bigquery.impl import BigQueryAdapter +from dbt.adapters.bigquery.relation import BigQueryRelation __all__ = [ BigQueryAdapter, + BigQueryRelation, ] diff --git a/dbt/adapters/snowflake/__init__.py b/dbt/adapters/snowflake/__init__.py index b1ce3942300..8ff756fdf4e 100644 --- a/dbt/adapters/snowflake/__init__.py +++ b/dbt/adapters/snowflake/__init__.py @@ -1,5 +1,7 @@ from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.snowflake.relation import SnowflakeRelation __all__ = [ SnowflakeAdapter, + SnowflakeRelation, ] From 87adc8a061e08e244d612705116afe748eb0a82b Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 23:21:44 -0400 Subject: [PATCH 23/61] table / view macros using relations --- TODO.md | 7 +++-- .../macros/adapters/bigquery.sql | 10 ++++--- .../global_project/macros/adapters/common.sql | 29 +++++++------------ .../macros/adapters/postgres.sql | 11 ------- .../macros/adapters/redshift.sql | 21 ++++++-------- .../macros/adapters/snowflake.sql | 18 ++---------- .../macros/materializations/helpers.sql | 8 ----- .../incremental/incremental.sql | 8 ++--- .../materializations/table/bigquery_table.sql | 12 ++++---- .../materializations/view/bigquery_view.sql | 6 ++-- .../macros/materializations/view/view.sql | 6 ++-- 11 files changed, 51 insertions(+), 85 deletions(-) delete mode 100644 dbt/include/global_project/macros/adapters/postgres.sql diff --git a/TODO.md b/TODO.md index 67424a89879..d65815d2df1 100644 --- a/TODO.md +++ b/TODO.md @@ -7,11 +7,14 @@ - [ ] get_columns_in_table - [x] rename - [x] truncate - - [ ] drop + - [x] drop - [ ] reset_csv_table - [ ] load_csv_rows - [ ] handle_csv_table - - [ ] macro: get_existing_relation_type + - [ ] make_date_partitioned_table + - [x] macro: get_existing_relation_type - [ ] macro: create_table_as + - [ ] macro: create_view_as + - [ ] macro: create_archive_table - (go back through these after BQ pr is merged) - [ ] deprecation warnings diff --git a/dbt/include/global_project/macros/adapters/bigquery.sql b/dbt/include/global_project/macros/adapters/bigquery.sql index be427b1d0d2..d1e4784fe69 100644 --- a/dbt/include/global_project/macros/adapters/bigquery.sql +++ b/dbt/include/global_project/macros/adapters/bigquery.sql @@ -10,18 +10,20 @@ {{ return(partition_by_clause) }} {%- endmacro -%} -{% macro bigquery__create_table_as(temporary, identifier, sql) -%} + +{% macro bigquery__create_table_as(temporary, relation, sql) -%} {%- set raw_partition_by = config.get('partition_by', none) -%} - create or replace table `{{ schema }}`.`{{ identifier }}` + create or replace table {{ relation }} {{ partition_by(raw_partition_by) }} as ( {{ sql }} ); {% endmacro %} -{% macro bigquery__create_view_as(identifier, sql) -%} - create or replace view `{{ schema }}`.`{{ identifier }}` as ( + +{% macro bigquery__create_view_as(relation, sql) -%} + create or replace view {{ relation }} as ( {{ sql }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 39cd75cf8d0..4d4e54c7c49 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -33,22 +33,25 @@ {% endmacro %} -{% macro create_table_as(temporary, identifier, sql) -%} - {{ adapter_macro('create_table_as', temporary, identifier, sql) }} +{% macro create_table_as(temporary, relation, sql) -%} + {{ adapter_macro('create_table_as', temporary, relation, sql) }} {%- endmacro %} -{% macro default__create_table_as(temporary, identifier, sql) -%} - create {% if temporary: -%}temporary{%- endif %} table {{ schema }}.{{ identifier }} as ( +{% macro default__create_table_as(temporary, relation, sql) -%} + create {% if temporary: -%}temporary{%- endif %} table + {{ relation.include(schema=(not temporary)) }} + as ( {{ sql }} ); {% endmacro %} -{% macro create_view_as(identifier, sql) -%} - {{ adapter_macro('create_view_as', identifier, sql) }} + +{% macro create_view_as(relation, sql) -%} + {{ adapter_macro('create_view_as', relation, sql) }} {%- endmacro %} -{% macro default__create_view_as(identifier, sql) -%} - create view {{ schema }}.{{ identifier }} as ( +{% macro default__create_view_as(relation, sql) -%} + create view {{ relation }} as ( {{ sql }} ); {% endmacro %} @@ -63,13 +66,3 @@ {{ column_list_for_create_table(columns) }} ); {% endmacro %} - - -{% macro get_existing_relation_type(existing, identifier) -%} - {{ return(adapter_macro('get_existing_relation_type', existing, identifier)) }} -{%- endmacro %} - -{% macro default__get_existing_relation_type(existing, identifier) -%} - {%- set existing_type = existing.get(identifier) -%} - {{ return(existing_type) }} -{%- endmacro %} diff --git a/dbt/include/global_project/macros/adapters/postgres.sql b/dbt/include/global_project/macros/adapters/postgres.sql deleted file mode 100644 index bfa106422dd..00000000000 --- a/dbt/include/global_project/macros/adapters/postgres.sql +++ /dev/null @@ -1,11 +0,0 @@ -{% macro postgres__create_table_as(temporary, relation, sql) -%} - {%- if temporary -%} - {%- set relation = relation.include(schema=False) -%} - {%- endif -%} - - create {% if temporary: -%}temporary{%- endif %} table - {{ relation }} - as ( - {{ sql }} - ); -{% endmacro %} diff --git a/dbt/include/global_project/macros/adapters/redshift.sql b/dbt/include/global_project/macros/adapters/redshift.sql index 24567ce12f5..b6696ca22ff 100644 --- a/dbt/include/global_project/macros/adapters/redshift.sql +++ b/dbt/include/global_project/macros/adapters/redshift.sql @@ -27,7 +27,7 @@ {%- endmacro -%} -{% macro redshift__create_table_as(temporary, identifier, sql) -%} +{% macro redshift__create_table_as(temporary, relation, sql) -%} {%- set _dist = config.get('dist') -%} {%- set _sort_type = config.get( @@ -37,31 +37,28 @@ 'sort', validator=validation.any[list, basestring]) -%} - {%- if temporary -%} - {%- set relation = relation.include(schema=False) -%} - {%- endif -%} - - create {% if temporary -%}temporary{%- endif %} table {{ relation }} - {{ dist(_dist) }} - {{ sort(_sort_type, _sort) }} + create {% if temporary -%}temporary{%- endif %} table + {{ relation.include(schema=(not temporary)) }} + {{ dist(_dist) }} + {{ sort(_sort_type, _sort) }} as ( {{ sql }} ); {%- endmacro %} -{% macro redshift__create_view_as(identifier, sql) -%} +{% macro redshift__create_view_as(relation, sql) -%} {% set bind_qualifier = '' if config.get('bind', default=True) else 'with no schema binding' %} - create view {{ schema }}.{{ identifier }} as ( + create view {{ relation }} as ( {{ sql }} ) {{ bind_qualifier }}; {% endmacro %} -{% macro redshift__create_archive_table(schema, identifier, columns) -%} - create table if not exists {{ schema }}.{{ identifier }} ( +{% macro redshift__create_archive_table(relation, columns) -%} + create table if not exists {{ relation }} ( {{ column_list_for_create_table(columns) }} ) {{ dist('dbt_updated_at') }} diff --git a/dbt/include/global_project/macros/adapters/snowflake.sql b/dbt/include/global_project/macros/adapters/snowflake.sql index 4040597008d..94c81e867fc 100644 --- a/dbt/include/global_project/macros/adapters/snowflake.sql +++ b/dbt/include/global_project/macros/adapters/snowflake.sql @@ -1,21 +1,7 @@ -{% macro snowflake__create_table_as(temporary, identifier, sql) -%} +{% macro snowflake__create_table_as(temporary, relation, sql) -%} {% if temporary %} use schema {{ schema }}; {% endif %} - create {% if temporary: -%}temporary{%- endif %} table - {% if not temporary: -%}{{ schema }}.{%- endif %}{{ identifier }} as ( - {{ sql }} - ); + {{ default__create_table_as(temporary, relation, sql) }} {% endmacro %} - - -{% macro snowflake__get_existing_relation_type(existing, identifier) -%} - {%- set upcased_existing = {} -%} - {%- for k,v in existing.items() -%} - {%- set _ = upcased_existing.update({k.upper(): v}) -%} - {%- endfor -%} - - {%- set existing_type = upcased_existing.get(identifier.upper()) -%} - {{ return(existing_type) }} -{%- endmacro %} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 33256fe0d4a..d816f46b5cc 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -45,14 +45,6 @@ {{ make_hook_config(sql, inside_transaction=False) }} {% endmacro %} - -{% macro drop_if_exists(existing, schema, name) %} - {% set existing_type = get_existing_relation_type(existing, name) %} - {% if existing_type is not none %} - {{ adapter.drop_relation(api.Relation.create(schema=schema, identifier=name, type=existing_type)) }} - {% endif %} -{% endmacro %} - {% macro drop_relation_if_exists(relation) %} {% if relation is not none %} {{ adapter.drop_relation(relation) }} diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql index 582b162ccd3..dd07dc842b6 100644 --- a/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -1,13 +1,13 @@ -{% macro dbt__incremental_delete(schema, model) -%} +{% macro dbt__incremental_delete(target_relation, tmp_relation) -%} {%- set unique_key = config.require('unique_key') -%} {%- set identifier = model['name'] -%} delete - from {{ schema }}.{{ identifier }} + from {{ target_relation }} where ({{ unique_key }}) in ( select ({{ unique_key }}) - from {{ identifier }}__dbt_incremental_tmp + from {{ tmp_relation }} ); {%- endmacro %} @@ -82,7 +82,7 @@ {% if unique_key is not none -%} - {{ dbt__incremental_delete(schema, model) }} + {{ dbt__incremental_delete(target_relation, tmp_relation) }} {%- endif %} diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql index 9885a6990d5..57070f9108c 100644 --- a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql +++ b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql @@ -1,7 +1,7 @@ -{% macro make_date_partitioned_table(model, dates, should_create, verbose=False) %} +{% macro make_date_partitioned_table(model, relation, dates, should_create, verbose=False) %} {% if should_create %} - {{ adapter.make_date_partitioned_table(model.schema, model.name) }} + {{ adapter.make_date_partitioned_table(relation.dataset, relation.identifier) }} {% endif %} {% for date in dates %} @@ -32,6 +32,8 @@ {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} {%- set existing_relations = adapter.list_relations(dataset=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} + {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- set target_relation = api.Relation(dataset=schema, identifier=identifier, type='table') -%} {%- set verbose = config.get('verbose', False) -%} {# partitions: iterate over each partition, running a separate query in a for-loop #} @@ -54,16 +56,16 @@ Since dbt uses WRITE_TRUNCATE mode for tables, we only need to drop this thing if it is not a table. If it _is_ already a table, then we can overwrite it without downtime #} - {%- if old_relation is not none and not old_relation.is_table -%} + {%- if exists_not_as_table -%} {{ adapter.drop_relation(old_relation) }} {%- endif -%} -- build model {% if partitions %} - {{ make_date_partitioned_table(model, partitions, (not old_relation.is_table), verbose) }} + {{ make_date_partitioned_table(model, target_relation, partitions, exists_not_as_table, verbose) }} {% else %} {% call statement('main') -%} - {{ create_table_as(False, identifier, sql) }} + {{ create_table_as(False, target_relation, sql) }} {% endcall -%} {% endif %} diff --git a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql index b3cd317c6bc..4b0590387c2 100644 --- a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql +++ b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql @@ -10,6 +10,8 @@ relations_list=existing_relations, schema=schema, identifier=identifier) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + {%- set target_relation = api.Relation.create( identifier=identifier, schema=schema, type='view') -%} @@ -29,14 +31,14 @@ {%- endif -%} -- build model - {% if old_relation is not none and old_relation.is_view and non_destructive_mode -%} + {% if exists_as_view and non_destructive_mode -%} {% call noop_statement('main', status="PASS", res=None) -%} -- Not running : non-destructive mode {{ sql }} {%- endcall %} {%- else -%} {% call statement('main') -%} - {{ create_view_as(identifier, sql) }} + {{ create_view_as(target_relation, sql) }} {%- endcall %} {%- endif %} diff --git a/dbt/include/global_project/macros/materializations/view/view.sql b/dbt/include/global_project/macros/materializations/view/view.sql index f49fc8017e2..a690206ad41 100644 --- a/dbt/include/global_project/macros/materializations/view/view.sql +++ b/dbt/include/global_project/macros/materializations/view/view.sql @@ -8,9 +8,9 @@ {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, - type='view') -%} + type='view') -%} {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, - schema=schema, type='view') -%} + schema=schema, type='view') -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} @@ -37,7 +37,7 @@ {%- endcall %} {%- else -%} {% call statement('main') -%} - {{ create_view_as(tmp_identifier, sql) }} + {{ create_view_as(intermediate_relation, sql) }} {%- endcall %} {%- endif %} From 79da833c607ec9b942f7a134bd9debdecca57f14 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 6 Apr 2018 23:36:46 -0400 Subject: [PATCH 24/61] fix some tests --- TODO.md | 6 +++--- dbt/adapters/default/impl.py | 1 + dbt/adapters/default/relation.py | 6 +++--- dbt/adapters/snowflake/relation.py | 8 ++++++++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/TODO.md b/TODO.md index d65815d2df1..d6545ae1abf 100644 --- a/TODO.md +++ b/TODO.md @@ -13,8 +13,8 @@ - [ ] handle_csv_table - [ ] make_date_partitioned_table - [x] macro: get_existing_relation_type - - [ ] macro: create_table_as - - [ ] macro: create_view_as - - [ ] macro: create_archive_table + - [x] macro: create_table_as + - [x] macro: create_view_as + - [x] macro: create_archive_table - (go back through these after BQ pr is merged) - [ ] deprecation warnings diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 4460faebbc1..8178a5a68a2 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -45,6 +45,7 @@ class DefaultAdapter(object): "get_relation", "drop_relation", "rename_relation", + "truncate_relation", "execute", "add_query", diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 55cb4c83202..8f2b48f99e3 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -154,9 +154,9 @@ def quoted(self, identifier): @classmethod def create_from_node(cls, profile, node, **kwargs): return cls.create( - database=profile['dbname'], - schema=node['schema'], - identifier=node['name'], + database=profile.get('dbname'), + schema=node.get('schema'), + identifier=node.get('name'), **kwargs) @classmethod diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index 00aa9471e8b..afff1af0ae4 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -20,6 +20,14 @@ class SnowflakeRelation(DefaultRelation): } } + @classmethod + def create_from_node(cls, profile, node, **kwargs): + return cls.create( + database=profile.get('database'), + schema=node.get('schema'), + identifier=node.get('name'), + **kwargs) + def matches(self, database=None, schema=None, identifier=None): search = filter_null_values({ 'database': database, From 82db36b70b2f6def5ad28b7f2c5ade0bf267cbdb Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sun, 8 Apr 2018 21:38:59 -0400 Subject: [PATCH 25/61] do not include schema if created as temp table --- .../global_project/macros/materializations/table/table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql index e8da50425fe..dc060bb9d08 100644 --- a/dbt/include/global_project/macros/materializations/table/table.sql +++ b/dbt/include/global_project/macros/materializations/table/table.sql @@ -43,7 +43,7 @@ insert into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} - from {{ intermediate_relation }} + from {{ intermediate_relation.include(schema=(not create_as_temporary)) }} ); {%- else -%} {{ create_table_as(create_as_temporary, target_relation, sql) }} From cadbad55860df3de96158f27edb1750edab5dc09 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sun, 8 Apr 2018 22:34:36 -0400 Subject: [PATCH 26/61] snowflake fixes --- dbt/adapters/snowflake/impl.py | 51 +++++++++++++++++++ dbt/adapters/snowflake/relation.py | 2 +- .../models/view_model.sql | 7 --- .../test_runtime_materialization.py | 12 ++--- 4 files changed, 58 insertions(+), 14 deletions(-) delete mode 100644 test/integration/017_runtime_materialization_tests/models/view_model.sql diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 708545d232c..7512acbfb5b 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -103,6 +103,31 @@ def open_connection(cls, connection): return result + @classmethod + def list_relations(cls, profile, model_name=None): + sql = """ + select + table_name as name, table_schema as schema, table_type as type + from information_schema.tables + """.strip() # noqa + + _, cursor = cls.add_query( + profile, sql, model_name, auto_begin=False) + + results = cursor.fetchall() + + relation_type_lookup = { + 'BASE TABLE': 'table', + 'VIEW': 'view' + + } + return [cls.Relation.create( + database=profile.get('database'), + schema=schema, + identifier=name, + type=relation_type_lookup.get(type)) + for (name, schema, type) in results] + @classmethod def query_for_existing(cls, profile, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): @@ -224,3 +249,29 @@ def cancel_connection(cls, profile, connection): res = cursor.fetchone() logger.debug("Cancel query '{}': {}".format(connection_name, res)) + + @classmethod + def _get_columns_in_table_sql(cls, schema_name, table_name, database): + + schema_filter = '1=1' + if schema_name is not None: + schema_filter = "table_schema ilike '{}'".format(schema_name) + + db_prefix = '' if database is None else '{}.'.format(database) + + sql = """ + select + column_name, + data_type, + character_maximum_length, + numeric_precision || ',' || numeric_scale as numeric_size + + from {db_prefix}information_schema.columns + where table_name ilike '{table_name}' + and {schema_filter} + order by ordinal_position + """.format(db_prefix=db_prefix, + table_name=table_name, + schema_filter=schema_filter).strip() + + return sql diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index afff1af0ae4..ff5f1cd6c1d 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -49,7 +49,7 @@ def matches(self, database=None, schema=None, identifier=None): return False else: - if self.get_path_part(k) == v.upper(): + if self.get_path_part(k) != v.upper(): return False return True diff --git a/test/integration/017_runtime_materialization_tests/models/view_model.sql b/test/integration/017_runtime_materialization_tests/models/view_model.sql deleted file mode 100644 index e0560b694c9..00000000000 --- a/test/integration/017_runtime_materialization_tests/models/view_model.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ - config( - materialized = "view" - ) -}} - -select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py index 4cdf83ac2ba..ae88bda04d0 100644 --- a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py +++ b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py @@ -42,7 +42,7 @@ def test_full_refresh(self): def test_non_destructive(self): self.run_dbt(['run', '--non-destructive']) - self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") self.assertTableDoesNotExist('dependent_view') @@ -52,7 +52,7 @@ def test_non_destructive(self): self.run_dbt(['run', '--non-destructive']) self.assertTableDoesExist('dependent_view') - self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -60,7 +60,7 @@ def test_non_destructive(self): def test_full_refresh_and_non_destructive(self): self.run_dbt(['run', '--full-refresh', '--non-destructive']) - self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") self.assertTableDoesNotExist('dependent_view') @@ -71,7 +71,7 @@ def test_full_refresh_and_non_destructive(self): self.run_dbt(['run', '--full-refresh', '--non-destructive']) self.assertTableDoesExist('dependent_view') - self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -80,7 +80,7 @@ def test_full_refresh_and_non_destructive(self): def test_delete__dbt_tmp_relation(self): # This creates a __dbt_tmp view - make sure it doesn't interfere with the dbt run self.run_sql_file("test/integration/017_runtime_materialization_tests/create_view__dbt_tmp.sql") - self.run_dbt(['run', '--model', 'view_model']) + self.run_dbt(['run', '--model', 'view']) self.assertTableDoesNotExist('view__model_dbt_tmp') - self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","view") From 614d067427a1b8dfcf3409b4f18c4ab65dfd2643 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sun, 8 Apr 2018 23:28:39 -0400 Subject: [PATCH 27/61] fix some tests --- dbt/adapters/bigquery/relation.py | 21 +++++++++++++++++++++ dbt/context/common.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index fd34509ed44..98bfa453c38 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -41,6 +41,27 @@ class BigQueryRelation(DefaultRelation): 'required': ['project', 'dataset', 'identifier'], } + SCHEMA = { + 'type': 'object', + 'properties': { + 'metadata': { + '_type': { + 'type': 'string', + 'const': 'DefaultRelation', + }, + }, + 'type': { + 'enum': DefaultRelation.RelationTypes + [None], + }, + 'path': PATH_SCHEMA, + 'include_policy': POLICY_SCHEMA, + 'quote_policy': POLICY_SCHEMA, + 'quote_character': {'type': 'string'}, + }, + 'required': ['metadata', 'type', 'path', 'include_policy', + 'quote_policy', 'quote_character'] + } + PATH_ELEMENTS = ['project', 'dataset', 'identifier'] def matches(self, project=None, dataset=None, identifier=None): diff --git a/dbt/context/common.py b/dbt/context/common.py index 3914fc29abf..13dccfc1ff1 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -270,7 +270,7 @@ def impl(message_if_exception, func, *args, **kwargs): def _return(value): - raise dbt.exceptions.MacroReturn(value) + raise dbt.exceptions.MacroReturn(value) def get_this_relation(db_wrapper, profile, model): From b7f948a001ce5b1f39f58d376944d3bc8ac588c0 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 01:39:42 -0400 Subject: [PATCH 28/61] fix bq relations --- dbt/adapters/bigquery/impl.py | 16 +++-- dbt/adapters/bigquery/relation.py | 70 +++++++++++++++---- .../materializations/table/bigquery_table.sql | 6 +- .../materializations/view/bigquery_view.sql | 8 +-- .../test_flattened_get_columns_in_table.sql | 5 +- .../test_get_columns_in_table.sql | 2 - 6 files changed, 73 insertions(+), 34 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index d57af9d14b9..94c40459bec 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -26,6 +26,7 @@ class BigQueryAdapter(PostgresAdapter): context_functions = [ + # deprecated -- use versions that take relations instead "query_for_existing", "execute_model", "drop", @@ -36,10 +37,17 @@ class BigQueryAdapter(PostgresAdapter): "already_exists", "expand_target_column_types", + # versions of adapter functions that take / return Relations + "list_relations", + "get_relation", + "drop_relation", + "rename_relation", + "get_columns_in_table" ] Relation = BigQueryRelation + Column = dbt.schema.BigQueryColumn SCOPE = ('https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform', @@ -47,8 +55,6 @@ class BigQueryAdapter(PostgresAdapter): QUERY_TIMEOUT = 300 - Column = dbt.schema.BigQueryColumn - @classmethod def handle_error(cls, error, message, sql): logger.debug(message.format(sql=sql)) @@ -173,7 +179,7 @@ def query_for_existing(cls, profile, schemas, model_name=None): for relation in all_relations} @classmethod - def list_relations(cls, profile, dataset, model_name=None): + def list_relations(cls, profile, schema, model_name=None): connection = cls.get_connection(profile, model_name) credentials = connection.get('credentials', {}) client = connection.get('handle') @@ -189,7 +195,7 @@ def list_relations(cls, profile, dataset, model_name=None): return [cls.Relation.create( project=credentials.get('project'), - dataset=dataset, + schema=schema, identifier=table.table_id, type=relation_types.get(table.table_type)) for table in all_tables] @@ -510,7 +516,7 @@ def create_csv_table(cls, profile, schema, table_name, agate_table): @classmethod def reset_csv_table(cls, profile, schema, table_name, agate_table, full_refresh=False, model_name=None): - relation = cls.Relation(dataset=schema, identifier=table_name) + relation = cls.Relation(schema=schema, identifier=table_name) cls.drop_relation(profile, relation, model_name) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 98bfa453c38..69f4686a941 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -8,15 +8,15 @@ class BigQueryRelation(DefaultRelation): 'metadata': { '_type': 'DefaultRelation' }, - 'quote_character': '"', + 'quote_character': '`', 'quote_policy': { 'project': True, - 'dataset': True, + 'schema': True, 'identifier': True }, 'include_policy': { - 'project': False, - 'dataset': True, + 'project': True, + 'schema': True, 'identifier': True } } @@ -25,20 +25,20 @@ class BigQueryRelation(DefaultRelation): 'type': 'object', 'properties': { 'project': {'type': ['string', 'null']}, - 'dataset': {'type': ['string', 'null']}, + 'schema': {'type': ['string', 'null']}, 'identifier': {'type': 'string'}, }, - 'required': ['project', 'dataset', 'identifier'], + 'required': ['project', 'schema', 'identifier'], } POLICY_SCHEMA = { 'type': 'object', 'properties': { 'project': {'type': 'boolean'}, - 'dataset': {'type': 'boolean'}, + 'schema': {'type': 'boolean'}, 'identifier': {'type': 'boolean'}, }, - 'required': ['project', 'dataset', 'identifier'], + 'required': ['project', 'schema', 'identifier'], } SCHEMA = { @@ -62,12 +62,36 @@ class BigQueryRelation(DefaultRelation): 'quote_policy', 'quote_character'] } - PATH_ELEMENTS = ['project', 'dataset', 'identifier'] + PATH_ELEMENTS = ['project', 'schema', 'identifier'] + + @classmethod + def create_from_node(cls, profile, node, **kwargs): + return cls.create( + project=profile.get('project'), + schema=node.get('schema'), + identifier=node.get('name'), + **kwargs) + + @classmethod + def create(cls, project=None, schema=None, + identifier=None, table_name=None, + type=None, **kwargs): + if table_name is None: + table_name = identifier + + return cls(type=type, + path={ + 'project': project, + 'schema': schema, + 'identifier': identifier + }, + table_name=table_name, + **kwargs) - def matches(self, project=None, dataset=None, identifier=None): + def matches(self, project=None, schema=None, identifier=None): search = filter_null_values({ 'project': project, - 'dataset': dataset, + 'schema': schema, 'identifier': identifier }) @@ -81,20 +105,36 @@ def matches(self, project=None, dataset=None, identifier=None): return True - def quote(self, project=None, dataset=None, identifier=None): + def quote(self, project=None, schema=None, identifier=None): policy = filter_null_values({ 'project': project, - 'dataset': dataset, + 'schema': schema, 'identifier': identifier }) return self.incorporate(quote_policy=policy) - def include(self, project=None, dataset=None, identifier=None): + def include(self, project=None, schema=None, identifier=None): policy = filter_null_values({ 'project': project, - 'dataset': dataset, + 'schema': schema, 'identifier': identifier }) return self.incorporate(include_policy=policy) + + @property + def project(self): + return self.path.get('project') + + @property + def schema(self): + return self.path.get('schema') + + @property + def schema(self): + return self.path.get('schema') + + @property + def identifier(self): + return self.path.get('identifier') diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql index 57070f9108c..42c19edfb06 100644 --- a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql +++ b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql @@ -1,7 +1,7 @@ {% macro make_date_partitioned_table(model, relation, dates, should_create, verbose=False) %} {% if should_create %} - {{ adapter.make_date_partitioned_table(relation.dataset, relation.identifier) }} + {{ adapter.make_date_partitioned_table(relation.schema, relation.identifier) }} {% endif %} {% for date in dates %} @@ -30,10 +30,10 @@ {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing_relations = adapter.list_relations(dataset=schema) -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} - {%- set target_relation = api.Relation(dataset=schema, identifier=identifier, type='table') -%} + {%- set target_relation = api.Relation(schema=schema, identifier=identifier, type='table') -%} {%- set verbose = config.get('verbose', False) -%} {# partitions: iterate over each partition, running a separate query in a for-loop #} diff --git a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql index 4b0590387c2..1ab47ea9c68 100644 --- a/dbt/include/global_project/macros/materializations/view/bigquery_view.sql +++ b/dbt/include/global_project/macros/materializations/view/bigquery_view.sql @@ -1,10 +1,9 @@ - {% materialization view, adapter='bigquery' -%} {%- set identifier = model['name'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing_relations = adapter.list_relations() -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation( relations_list=existing_relations, @@ -16,11 +15,6 @@ identifier=identifier, schema=schema, type='view') -%} - {%- set intermediate_relation = api.Relation.create( - identifier=tmp_identifier, - schema=schema, type='view') -%} - - -- drop if exists {%- if old_relation is not none -%} {%- if old_relation.is_table and not flags.FULL_REFRESH -%} diff --git a/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql b/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql index 1a741dbd102..330e0d67111 100644 --- a/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql +++ b/test/integration/022_bigquery_test/adapter-models/test_flattened_get_columns_in_table.sql @@ -1,8 +1,9 @@ - - {% set source = ref('source') %} {% set cols = adapter.get_columns_in_table(source.schema, source.name) %} +{{ log('source') }} +{{ log(source) }} + {% set flattened = [] %} {% for col in cols %} {% if col.mode == 'REPEATED' %} diff --git a/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql b/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql index 3653fee869c..58503c93b92 100644 --- a/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql +++ b/test/integration/022_bigquery_test/adapter-models/test_get_columns_in_table.sql @@ -1,5 +1,3 @@ - - {% set source = ref('source') %} {% set cols = adapter.get_columns_in_table(source.schema, source.name) %} From a8db16a8ead7694ea2a79a16a4bb17a37b505b85 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 01:51:51 -0400 Subject: [PATCH 29/61] bq getting closer --- dbt/adapters/bigquery/impl.py | 11 +++-- dbt/adapters/bigquery/relation.py | 40 +++++++++---------- dbt/adapters/snowflake/relation.py | 23 ++++++++++- .../materializations/table/bigquery_table.sql | 2 +- 4 files changed, 48 insertions(+), 28 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 94c40459bec..ab6e69ff53e 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -184,7 +184,7 @@ def list_relations(cls, profile, schema, model_name=None): credentials = connection.get('credentials', {}) client = connection.get('handle') - bigquery_dataset = cls.get_dataset(profile, dataset, model_name) + bigquery_dataset = cls.get_dataset(profile, schema, model_name) all_tables = client.list_tables(bigquery_dataset) relation_types = { @@ -431,10 +431,8 @@ def get_columns_in_table(cls, profile, schema_name, table_name, columns = [] for col in table_schema: - name = col.name - data_type = col.field_type - - column = cls.Column(col.name, col.field_type, col.fields, col.mode) + column = cls.Column( + col.name, col.field_type, col.fields, col.mode) columns.append(column) return columns @@ -516,7 +514,8 @@ def create_csv_table(cls, profile, schema, table_name, agate_table): @classmethod def reset_csv_table(cls, profile, schema, table_name, agate_table, full_refresh=False, model_name=None): - relation = cls.Relation(schema=schema, identifier=table_name) + relation = cls.Relation.create(schema=schema, + identifier=table_name) cls.drop_relation(profile, relation, model_name) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 69f4686a941..bfd366d02e3 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -6,7 +6,7 @@ class BigQueryRelation(DefaultRelation): DEFAULTS = { 'metadata': { - '_type': 'DefaultRelation' + '_type': 'BigQueryRelation' }, 'quote_character': '`', 'quote_policy': { @@ -47,7 +47,7 @@ class BigQueryRelation(DefaultRelation): 'metadata': { '_type': { 'type': 'string', - 'const': 'DefaultRelation', + 'const': 'BigQueryRelation', }, }, 'type': { @@ -64,6 +64,23 @@ class BigQueryRelation(DefaultRelation): PATH_ELEMENTS = ['project', 'schema', 'identifier'] + def matches(self, project=None, schema=None, identifier=None): + search = filter_null_values({ + 'project': project, + 'schema': schema, + 'identifier': identifier + }) + + if not search: + # nothing was passed in + pass + + for k, v in search.items(): + if self.get_path_part(k) != v: + return False + + return True + @classmethod def create_from_node(cls, profile, node, **kwargs): return cls.create( @@ -88,23 +105,6 @@ def create(cls, project=None, schema=None, table_name=table_name, **kwargs) - def matches(self, project=None, schema=None, identifier=None): - search = filter_null_values({ - 'project': project, - 'schema': schema, - 'identifier': identifier - }) - - if not search: - # nothing was passed in - pass - - for k, v in search.items(): - if self.get_path_part(k) != v: - return False - - return True - def quote(self, project=None, schema=None, identifier=None): policy = filter_null_values({ 'project': project, @@ -132,7 +132,7 @@ def schema(self): return self.path.get('schema') @property - def schema(self): + def dataset(self): return self.path.get('schema') @property diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index ff5f1cd6c1d..1db87909ead 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -5,7 +5,7 @@ class SnowflakeRelation(DefaultRelation): DEFAULTS = { 'metadata': { - '_type': 'DefaultRelation' + '_type': 'SnowflakeRelation' }, 'quote_character': '"', 'quote_policy': { @@ -20,6 +20,27 @@ class SnowflakeRelation(DefaultRelation): } } + SCHEMA = { + 'type': 'object', + 'properties': { + 'metadata': { + '_type': { + 'type': 'string', + 'const': 'SnowflakeRelation', + }, + }, + 'type': { + 'enum': DefaultRelation.RelationTypes + [None], + }, + 'path': DefaultRelation.PATH_SCHEMA, + 'include_policy': DefaultRelation.POLICY_SCHEMA, + 'quote_policy': DefaultRelation.POLICY_SCHEMA, + 'quote_character': {'type': 'string'}, + }, + 'required': ['metadata', 'type', 'path', 'include_policy', + 'quote_policy', 'quote_character'] + } + @classmethod def create_from_node(cls, profile, node, **kwargs): return cls.create( diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql index 42c19edfb06..2c3db882381 100644 --- a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql +++ b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql @@ -33,7 +33,7 @@ {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, identifier=identifier) -%} {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} - {%- set target_relation = api.Relation(schema=schema, identifier=identifier, type='table') -%} + {%- set target_relation = api.Relation.create(schema=schema, identifier=identifier, type='table') -%} {%- set verbose = config.get('verbose', False) -%} {# partitions: iterate over each partition, running a separate query in a for-loop #} From 58c271212290ec7734d337aa0fec8a65d7f5933c Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 02:05:07 -0400 Subject: [PATCH 30/61] archive macros --- dbt/include/global_project/macros/adapters/common.sql | 8 ++++---- .../macros/materializations/archive/archive.sql | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 4d4e54c7c49..7d6a42dba1d 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -57,12 +57,12 @@ {% endmacro %} -{% macro create_archive_table(schema, identifier, columns) -%} - {{ adapter_macro('create_archive_table', schema, identifier, columns) }} +{% macro create_archive_table(relation, columns) -%} + {{ adapter_macro('create_archive_table', relation, columns) }} {%- endmacro %} -{% macro default__create_archive_table(schema, identifier, columns) -%} - create table if not exists {{ schema }}.{{ identifier }} ( +{% macro default__create_archive_table(relation, columns) -%} + create table if not exists {{ relation }} ( {{ column_list_for_create_table(columns) }} ); {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/dbt/include/global_project/macros/materializations/archive/archive.sql index c1977bf3f8b..fce49d6f940 100644 --- a/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -117,7 +117,7 @@ {% endcall %} {% call statement() %} - {{ create_archive_table(target_schema, target_table, dest_columns) }} + {{ create_archive_table(target_relation, dest_columns) }} {% endcall %} {% set missing_columns = adapter.get_missing_columns(source_schema, source_table, target_schema, target_table) %} From d4b2516d792bc41948f99767f1596f2c80e764c2 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 02:37:29 -0400 Subject: [PATCH 31/61] fixing postgres tests --- dbt/adapters/default/relation.py | 4 ++-- dbt/context/common.py | 5 +++-- .../013_context_var_tests/test_context_vars.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 8f2b48f99e3..831939b5f36 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -152,11 +152,11 @@ def quoted(self, identifier): identifier=identifier) @classmethod - def create_from_node(cls, profile, node, **kwargs): + def create_from_node(cls, profile, node, table_name=None, **kwargs): return cls.create( database=profile.get('dbname'), schema=node.get('schema'), - identifier=node.get('name'), + identifier=table_name or node.get('name'), **kwargs) @classmethod diff --git a/dbt/context/common.py b/dbt/context/common.py index 13dccfc1ff1..56ced6044fb 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -276,8 +276,9 @@ def _return(value): def get_this_relation(db_wrapper, profile, model): table_name = dbt.utils.model_immediate_name( model, dbt.flags.NON_DESTRUCTIVE) - return db_wrapper.adapter.Relation.create_from_node(profile, model, - table_name=table_name) + + return db_wrapper.adapter.Relation.create_from_node( + profile, model, table_name=table_name) def generate(model, project, flat_graph, provider=None): diff --git a/test/integration/013_context_var_tests/test_context_vars.py b/test/integration/013_context_var_tests/test_context_vars.py index 315da91be77..4b92ea480ae 100644 --- a/test/integration/013_context_var_tests/test_context_vars.py +++ b/test/integration/013_context_var_tests/test_context_vars.py @@ -1,9 +1,9 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest -import dbt.flags import os + class TestContextVars(DBTIntegrationTest): def setUp(self): @@ -89,7 +89,10 @@ def test_env_vars_dev(self): self.run_dbt(['run']) ctx = self.get_ctx_vars() - self.assertEqual(ctx['this'], '{}.context__dbt_tmp'.format(self.unique_schema())) + self.assertEqual( + ctx['this'], + '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual(ctx['this.name'], 'context') self.assertEqual(ctx['this.schema'], self.unique_schema()) self.assertEqual(ctx['this.table'], 'context__dbt_tmp') @@ -111,7 +114,10 @@ def test_env_vars_prod(self): self.run_dbt(['run', '--target', 'prod']) ctx = self.get_ctx_vars() - self.assertEqual(ctx['this'], '{}.context__dbt_tmp'.format(self.unique_schema())) + self.assertEqual( + ctx['this'], + '"{}"."context__dbt_tmp"'.format(self.unique_schema())) + self.assertEqual(ctx['this.name'], 'context') self.assertEqual(ctx['this.schema'], self.unique_schema()) self.assertEqual(ctx['this.table'], 'context__dbt_tmp') From cea3a4391343c034c65e18a1da0cc9118f1cf2c2 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 02:56:45 -0400 Subject: [PATCH 32/61] fix postgres tests --- .../017_runtime_materialization_tests/models/view.sql | 7 +++++++ .../test_runtime_materialization.py | 4 ++-- .../017_runtime_materialization_tests/update.sql | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 test/integration/017_runtime_materialization_tests/models/view.sql diff --git a/test/integration/017_runtime_materialization_tests/models/view.sql b/test/integration/017_runtime_materialization_tests/models/view.sql new file mode 100644 index 00000000000..e0560b694c9 --- /dev/null +++ b/test/integration/017_runtime_materialization_tests/models/view.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized = "view" + ) +}} + +select * from {{ this.schema }}.seed diff --git a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py index ae88bda04d0..95f5f136219 100644 --- a/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py +++ b/test/integration/017_runtime_materialization_tests/test_runtime_materialization.py @@ -21,7 +21,7 @@ def test_full_refresh(self): # initial full-refresh should have no effect self.run_dbt(['run', '--full-refresh']) - self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -34,7 +34,7 @@ def test_full_refresh(self): self.run_dbt(['run', '--full-refresh']) - self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") diff --git a/test/integration/017_runtime_materialization_tests/update.sql b/test/integration/017_runtime_materialization_tests/update.sql index 5ded8d56287..f5a37926d81 100644 --- a/test/integration/017_runtime_materialization_tests/update.sql +++ b/test/integration/017_runtime_materialization_tests/update.sql @@ -6,7 +6,7 @@ create view {schema}.dependent_view as ( select count(*) from {schema}.materialized union all - select count(*) from {schema}.view_model + select count(*) from {schema}.view union all select count(*) from {schema}.incremental From 5ce7aa7f19701935517a894b3d5a123486bb756c Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 9 Apr 2018 15:38:47 -0400 Subject: [PATCH 33/61] Implement relations api dp fix (#729) * fix bq date partitioning * fix for context vars (tmp table name) --- dbt/adapters/default/impl.py | 2 +- dbt/adapters/default/relation.py | 7 +++++-- .../macros/materializations/table/bigquery_table.sql | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 8178a5a68a2..39740155d8c 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -695,7 +695,7 @@ def drop_schema(cls, profile, schema, model_name=None): @classmethod def already_exists(cls, profile, schema, table, model_name=None): # TODO add deprecation warning - relation = cls.get_relation(schema=schema, identifier=table) + relation = cls.get_relation(profile, schema=schema, identifier=table) return relation is not None @classmethod diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 831939b5f36..3f153dffc65 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -127,6 +127,8 @@ def render(self): if path_part is None: continue + elif k == 'identifier': + path_part = self.table parts.append( self.quote_if( @@ -156,7 +158,8 @@ def create_from_node(cls, profile, node, table_name=None, **kwargs): return cls.create( database=profile.get('dbname'), schema=node.get('schema'), - identifier=table_name or node.get('name'), + identifier=node.get('name'), + table_name=table_name, **kwargs) @classmethod @@ -205,7 +208,7 @@ def name(self): # Here for compatibility with old Relation interface @property def table(self): - return self._table_name + return self.table_name @property def is_table(self): diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql index 2c3db882381..3c50969bbab 100644 --- a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql +++ b/dbt/include/global_project/macros/materializations/table/bigquery_table.sql @@ -62,7 +62,9 @@ -- build model {% if partitions %} - {{ make_date_partitioned_table(model, target_relation, partitions, exists_not_as_table, verbose) }} + {# Create the dp-table if 1. it does not exist or 2. it existed, but we just dropped it #} + {%- set should_create = (old_relation is none or exists_not_as_table) -%} + {{ make_date_partitioned_table(model, target_relation, partitions, should_create, verbose) }} {% else %} {% call statement('main') -%} {{ create_table_as(False, target_relation, sql) }} From a466b81f299228716c164e6ee902b311ddd0e642 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 19:44:07 -0400 Subject: [PATCH 34/61] address code review comments --- dbt/adapters/bigquery/impl.py | 4 ++-- dbt/adapters/default/impl.py | 5 +++++ dbt/exceptions.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index ab6e69ff53e..60f5aa1ad23 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -205,8 +205,8 @@ def drop_relation(cls, profile, relation, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, relation._schema, model_name) - relation_object = dataset.table(relation._identifier) + dataset = cls.get_dataset(profile, relation.schema, model_name) + relation_object = dataset.table(relation.identifier) client.delete_table(relation_object) @classmethod diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 39740155d8c..f332e064d15 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -162,6 +162,11 @@ def drop(cls, profile, schema, relation, relation_type, model_name=None): @classmethod def drop_relation(cls, profile, relation, model_name=None): + if relation.type is None: + dbt.exceptions.raise_compiler_error( + 'Tried to drop relation {}, but its type is null.' + .format(relation)) + sql = 'drop {} if exists {} cascade'.format(relation.type, relation) connection, cursor = cls.add_query(profile, sql, model_name) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index fd11fc80f73..ee9fd82ba64 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -256,7 +256,7 @@ def missing_relation(relation, model=None): def relation_wrong_type(relation, expected_type, model=None): raise_compiler_error( - ('Trying to create {intended_type} {relation}, ' + ('Trying to create {expected_type} {relation}, ' 'but it currently exists as a {current_type}. Either ' 'drop {relation} manually, or run dbt with ' '`--full-refresh` and dbt will drop it for you.') From 2f4dcb958c1600ce8ec908fe572d38e09a4fab44 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 20:20:07 -0400 Subject: [PATCH 35/61] deprecations --- dbt/adapters/bigquery/impl.py | 9 +++++++++ dbt/adapters/default/impl.py | 37 ++++++++++++++++++++++++++-------- dbt/adapters/snowflake/impl.py | 5 +++++ dbt/deprecations.py | 11 +++++----- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 60f5aa1ad23..995c843443d 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -3,6 +3,7 @@ from contextlib import contextmanager import dbt.compat +import dbt.deprecations import dbt.exceptions import dbt.schema import dbt.flags as flags @@ -166,6 +167,10 @@ def open_connection(cls, connection): @classmethod def query_for_existing(cls, profile, schemas, model_name=None): + dbt.deprecations.warn('relations-api', + fn='query_for_existing', + model=model_name) + if not isinstance(schemas, (list, tuple)): schemas = [schemas] @@ -479,6 +484,10 @@ def quote(cls, identifier): @classmethod def quote_schema_and_table(cls, profile, schema, table, model_name=None): + dbt.deprecations.warn('relations-api', + fn='quote_schema_and_table', + model=model_name) + return cls.render_relation(profile, cls.quote(schema), cls.quote(table)) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index f332e064d15..44171e1da37 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -5,6 +5,7 @@ from contextlib import contextmanager +import dbt.deprecations import dbt.exceptions import dbt.flags import dbt.schema @@ -27,15 +28,16 @@ class DefaultAdapter(object): requires = {} context_functions = [ - # deprecated -- use versions that take relations instead - "already_exists", "get_columns_in_table", "get_missing_columns", + "expand_target_column_types", + + # deprecated -- use versions that take relations instead + "already_exists", "query_for_existing", "rename", "drop", "truncate", - "expand_target_column_types", # just deprecated. going away in a future release "quote_schema_and_table", @@ -94,6 +96,10 @@ def alter_column_type(cls, profile, schema, table, column_name, @classmethod def query_for_existing(cls, profile, schemas, model_name=None): + dbt.deprecations.warn('relations-api', + fn='query_for_existing', + model=model_name) + if not isinstance(schemas, (list, tuple)): schemas = [schemas] @@ -151,7 +157,10 @@ def get_result_from_cursor(cls, cursor): @classmethod def drop(cls, profile, schema, relation, relation_type, model_name=None): - # add deprecation warning + dbt.deprecations.warn('relations-api', + fn='drop', + model=model_name) + identifier = relation relation = cls.Relation.create( schema=schema, @@ -173,7 +182,10 @@ def drop_relation(cls, profile, relation, model_name=None): @classmethod def truncate(cls, profile, schema, table, model_name=None): - # add deprecation warning + dbt.deprecations.warn('relations-api', + fn='truncate', + model=model_name) + relation = cls.Relation.create( schema=schema, identifier=table, @@ -189,7 +201,10 @@ def truncate_relation(cls, profile, relation, model_name=None): @classmethod def rename(cls, profile, schema, from_name, to_name, model_name=None): - # add deprecation warning + dbt.deprecations.warn('relations-api', + fn='rename', + model=model_name) + return cls.rename_relation( profile, from_relation=cls.Relation.create( @@ -260,7 +275,6 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): @classmethod def get_columns_in_table(cls, profile, schema_name, table_name, database=None, model_name=None): - sql = cls._get_columns_in_table_sql(schema_name, table_name, database) connection, cursor = cls.add_query( profile, sql, model_name) @@ -699,7 +713,10 @@ def drop_schema(cls, profile, schema, model_name=None): @classmethod def already_exists(cls, profile, schema, table, model_name=None): - # TODO add deprecation warning + dbt.deprecations.warn('relations-api', + fn='already_exists', + model=model_name) + relation = cls.get_relation(profile, schema=schema, identifier=table) return relation is not None @@ -709,6 +726,10 @@ def quote(cls, identifier): @classmethod def quote_schema_and_table(cls, profile, schema, table, model_name=None): + dbt.deprecations.warn('relations-api', + fn='quote_schema_and_table', + model=model_name) + return '{}.{}'.format(cls.quote(schema), cls.quote(table)) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index d4dbb683af9..8b7369b1023 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -9,6 +9,7 @@ from contextlib import contextmanager import dbt.compat +import dbt.deprecations import dbt.exceptions from dbt.adapters.postgres import PostgresAdapter @@ -131,6 +132,10 @@ def list_relations(cls, profile, model_name=None): @classmethod def query_for_existing(cls, profile, schemas, model_name=None): + dbt.deprecations.warn('relations-api', + fn='query_for_existing', + model=model_name) + if not isinstance(schemas, (list, tuple)): schemas = [schemas] diff --git a/dbt/deprecations.py b/dbt/deprecations.py index 72ac37597af..d4ade57fb63 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -33,13 +33,14 @@ class SeedDropExistingDeprecation(DBTDeprecation): will be removed in a future version of dbt.""" -class OldStyleAdapterFunctionsDeprecations(DBTDeprecation): +class NonRelationAdapterFunctionDeprecation(DBTDeprecation): # TODO add link to docs - name = 'old-style-adapter-functions' + name = 'relations-api' description = """ - This is an old-style adapter function (takes a schema and identifier - instead of a Relation object). Please use the new-style functions. - You can find more information in the documentation. (ADD LINK) + Your project uses a deprecated adapter function '{fn}'. + Please use the Relations API instead. You can find + more information in the documentation. + (ADD LINK) """ From cd8df156566d8a85d06b6cdd89b14e4345e6aad1 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 9 Apr 2018 21:06:46 -0400 Subject: [PATCH 36/61] fix tests --- dbt/deprecations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index d4ade57fb63..d497e7c3f34 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -62,7 +62,7 @@ def warn(name, *args, **kwargs): deprecations_list = [ DBTRepositoriesDeprecation(), SeedDropExistingDeprecation(), - OldStyleAdapterFunctionsDeprecations(), + NonRelationAdapterFunctionDeprecation(), ] deprecations = {d.name: d for d in deprecations_list} From c1cfc316a65461a19da93f8cd8d427f2c2018ad9 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 10 Apr 2018 13:12:15 -0400 Subject: [PATCH 37/61] make relations hashable for compatibility with existing dbt code --- dbt/adapters/default/relation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 3f153dffc65..17d380facbe 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -181,6 +181,9 @@ def create(cls, database=None, schema=None, def __repr__(self): return "<{} {}>".format(self.__class__.__name__, self.render()) + def __hash__(self): + return hash(self.render()) + def __str__(self): return self.render() From 596e4333d04a3f1e7fdc5b8f48853b9ca33a70eb Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 10 Apr 2018 17:46:38 -0400 Subject: [PATCH 38/61] rip out deprecations, make api more consistent --- dbt/adapters/bigquery/impl.py | 22 -------- dbt/adapters/default/impl.py | 52 ++++++------------- dbt/adapters/postgres/impl.py | 10 ++-- dbt/adapters/snowflake/impl.py | 14 ++--- .../materializations/archive/archive.sql | 3 -- .../incremental/incremental.sql | 2 +- .../macros/materializations/table/table.sql | 2 +- .../macros/materializations/view/view.sql | 2 +- 8 files changed, 29 insertions(+), 78 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 995c843443d..3f014eef951 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -165,24 +165,6 @@ def open_connection(cls, connection): result['state'] = 'open' return result - @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): - dbt.deprecations.warn('relations-api', - fn='query_for_existing', - model=model_name) - - if not isinstance(schemas, (list, tuple)): - schemas = [schemas] - - all_relations = [] - - for schema in schemas: - all_relations.extend(cls.list_relations( - profile, schema, model_name)) - - return {relation.identifier: relation.type - for relation in all_relations} - @classmethod def list_relations(cls, profile, schema, model_name=None): connection = cls.get_connection(profile, model_name) @@ -484,10 +466,6 @@ def quote(cls, identifier): @classmethod def quote_schema_and_table(cls, profile, schema, table, model_name=None): - dbt.deprecations.warn('relations-api', - fn='quote_schema_and_table', - model=model_name) - return cls.render_relation(profile, cls.quote(schema), cls.quote(table)) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 44171e1da37..42f0255dc0d 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -5,7 +5,6 @@ from contextlib import contextmanager -import dbt.deprecations import dbt.exceptions import dbt.flags import dbt.schema @@ -96,18 +95,17 @@ def alter_column_type(cls, profile, schema, table, column_name, @classmethod def query_for_existing(cls, profile, schemas, model_name=None): - dbt.deprecations.warn('relations-api', - fn='query_for_existing', - model=model_name) - if not isinstance(schemas, (list, tuple)): schemas = [schemas] - all_relations = cls.list_relations(profile, model_name) + all_relations = [] + + for schema in schemas: + all_relations.extend( + cls.list_relations(profile, schema, model_name)) return {relation.identifier: relation.type - for relation in all_relations - if relation.schema in schemas} + for relation in all_relations} @classmethod def get_existing_schemas(cls, profile, model_name=None): @@ -157,10 +155,6 @@ def get_result_from_cursor(cls, cursor): @classmethod def drop(cls, profile, schema, relation, relation_type, model_name=None): - dbt.deprecations.warn('relations-api', - fn='drop', - model=model_name) - identifier = relation relation = cls.Relation.create( schema=schema, @@ -182,10 +176,6 @@ def drop_relation(cls, profile, relation, model_name=None): @classmethod def truncate(cls, profile, schema, table, model_name=None): - dbt.deprecations.warn('relations-api', - fn='truncate', - model=model_name) - relation = cls.Relation.create( schema=schema, identifier=table, @@ -201,10 +191,6 @@ def truncate_relation(cls, profile, relation, model_name=None): @classmethod def rename(cls, profile, schema, from_name, to_name, model_name=None): - dbt.deprecations.warn('relations-api', - fn='rename', - model=model_name) - return cls.rename_relation( profile, from_relation=cls.Relation.create( @@ -327,20 +313,25 @@ def expand_target_column_types(cls, profile, # RELATIONS ### @classmethod - def list_relations(cls, profile, model_name=None): + def list_relations(cls, profile, schema, model_name=None): raise dbt.exceptions.NotImplementedException( '`list_relations` is not implemented for this adapter!') @classmethod - def get_relation(cls, profile, relations_list=None, + def get_relation(cls, profile, schema=None, relations_list=None, model_name=None, **kwargs): + if schema is None and relations_list is None: + raise dbt.exceptions.RuntimeException( + 'get_relation needs either a schema to query, or a list ' + 'of relations to use') + if relations_list is None: - relations_list = cls.list_relations(profile, model_name) + relations_list = cls.list_relations(profile, schema, model_name) matches = [] for relation in relations_list: - if relation.matches(**kwargs): + if relation.matches(schema=schema, **kwargs): matches.append(relation) if len(matches) > 1: @@ -352,11 +343,6 @@ def get_relation(cls, profile, relations_list=None, return None - @classmethod - def reload_relation(cls, profile, relation, model_name=None): - return cls.get_relation(profile, model_name=model_name, - **filter_null_values(relation.path)) - ### # SANE ANSI SQL DEFAULTS ### @@ -713,10 +699,6 @@ def drop_schema(cls, profile, schema, model_name=None): @classmethod def already_exists(cls, profile, schema, table, model_name=None): - dbt.deprecations.warn('relations-api', - fn='already_exists', - model=model_name) - relation = cls.get_relation(profile, schema=schema, identifier=table) return relation is not None @@ -726,10 +708,6 @@ def quote(cls, identifier): @classmethod def quote_schema_and_table(cls, profile, schema, table, model_name=None): - dbt.deprecations.warn('relations-api', - fn='quote_schema_and_table', - model=model_name) - return '{}.{}'.format(cls.quote(schema), cls.quote(table)) diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 8b53a9862a6..a8bff4af85e 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -115,12 +115,14 @@ def alter_column_type(cls, profile, schema, table, column_name, return connection, cursor @classmethod - def list_relations(cls, profile, model_name=None): + def list_relations(cls, profile, schema, model_name=None): sql = """ select tablename as name, schemaname as schema, 'table' as type from pg_tables + where schemaname = '{schema}' union all select viewname as name, schemaname as schema, 'view' as type from pg_views - """.strip() # noqa + where schemaname = '{schema}' + """.format(schema=schema).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name, auto_begin=False) @@ -129,10 +131,10 @@ def list_relations(cls, profile, model_name=None): return [cls.Relation.create( database=profile.get('dbname'), - schema=schema, + schema=_schema, identifier=name, type=type) - for (name, schema, type) in results] + for (name, _schema, type) in results] @classmethod def get_existing_schemas(cls, profile, model_name=None): diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 8b7369b1023..376000e9d64 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -9,7 +9,6 @@ from contextlib import contextmanager import dbt.compat -import dbt.deprecations import dbt.exceptions from dbt.adapters.postgres import PostgresAdapter @@ -106,12 +105,13 @@ def open_connection(cls, connection): return result @classmethod - def list_relations(cls, profile, model_name=None): + def list_relations(cls, profile, schema, model_name=None): sql = """ select table_name as name, table_schema as schema, table_type as type from information_schema.tables - """.strip() # noqa + where table_schema ilike '{schema}' + """.format(schema=schema).strip() # noqa _, cursor = cls.add_query( profile, sql, model_name, auto_begin=False) @@ -125,17 +125,13 @@ def list_relations(cls, profile, model_name=None): } return [cls.Relation.create( database=profile.get('database'), - schema=schema, + schema=_schema, identifier=name, type=relation_type_lookup.get(type)) - for (name, schema, type) in results] + for (name, _schema, type) in results] @classmethod def query_for_existing(cls, profile, schemas, model_name=None): - dbt.deprecations.warn('relations-api', - fn='query_for_existing', - model=model_name) - if not isinstance(schemas, (list, tuple)): schemas = [schemas] diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/dbt/include/global_project/macros/materializations/archive/archive.sql index fce49d6f940..738c2465130 100644 --- a/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -72,18 +72,15 @@ {% materialization archive, default %} {%- set config = model['config'] -%} - {%- set existing_relations = adapter.list_relations() -%} {%- set target_schema = config.get('target_schema') -%} {%- set target_table = config.get('target_table') -%} {%- set source_relation = adapter.get_relation( - relations_list=existing_relations, schema=config.get('source_schema'), identifier=config.get('source_table')) -%} {%- set target_relation = adapter.get_relation( - relations_list=existing_relations, schema=target_schema, identifier=target_table) -%} diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql index dd07dc842b6..0bf8c7bff30 100644 --- a/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -19,7 +19,7 @@ {%- set identifier = model['name'] -%} {%- set tmp_identifier = model['name'] + '__dbt_incremental_tmp' -%} - {%- set existing_relations = adapter.list_relations() -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql index dc060bb9d08..b98f6e3f95d 100644 --- a/dbt/include/global_project/macros/materializations/table/table.sql +++ b/dbt/include/global_project/macros/materializations/table/table.sql @@ -3,7 +3,7 @@ {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing_relations = adapter.list_relations() -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} diff --git a/dbt/include/global_project/macros/materializations/view/view.sql b/dbt/include/global_project/macros/materializations/view/view.sql index a690206ad41..1f0036116bc 100644 --- a/dbt/include/global_project/macros/materializations/view/view.sql +++ b/dbt/include/global_project/macros/materializations/view/view.sql @@ -4,7 +4,7 @@ {%- set tmp_identifier = identifier + '__dbt_tmp' -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set existing_relations = adapter.list_relations() -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, From 533f5c378140b1ceda2b48857fd4603ffec4b196 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 11 Apr 2018 10:40:36 -0400 Subject: [PATCH 39/61] actually fix merge conflict --- test/integration/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/integration/base.py b/test/integration/base.py index 178335dac10..134b0a031cb 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -251,12 +251,8 @@ def tearDown(self): if self.adapter_type == 'bigquery': adapter.drop_schema(self.profile, self.unique_schema(), '__test') else: -<<<<<<< HEAD - self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) -======= - self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE' + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE' .format(self.unique_schema())) ->>>>>>> d966ec28aae4cc70b7da409ae708cf29ff3d52cd self.handle.close() # hack for BQ -- TODO From 9fe9ca86f836e2c5e7ad083894cb79913aa5800b Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sun, 22 Apr 2018 15:23:59 -0400 Subject: [PATCH 40/61] global quoting config --- dbt/adapters/bigquery/impl.py | 4 ++++ dbt/adapters/postgres/impl.py | 8 ++++++-- dbt/adapters/snowflake/impl.py | 4 ++++ dbt/context/common.py | 15 ++++++++++++++- dbt/context/parser.py | 4 +++- dbt/context/runtime.py | 4 +++- dbt/contracts/graph/parsed.py | 1 + .../macros/materializations/table/table.sql | 5 +++-- dbt/main.py | 1 + dbt/model.py | 5 +++-- dbt/project.py | 18 ++++++++++++++++++ dbt/utils.py | 1 + 12 files changed, 61 insertions(+), 9 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index e4f6cf8e1f4..33538088f90 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -193,6 +193,10 @@ def list_relations(cls, profile, schema, model_name=None): project=credentials.get('project'), schema=schema, identifier=table.table_id, + quote_policy={ + 'schema': True, + 'identifier': True + }, type=relation_types.get(table.table_type)) for table in all_tables] diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index a8bff4af85e..405bc435934 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -118,10 +118,10 @@ def alter_column_type(cls, profile, schema, table, column_name, def list_relations(cls, profile, schema, model_name=None): sql = """ select tablename as name, schemaname as schema, 'table' as type from pg_tables - where schemaname = '{schema}' + where schemaname ilike '{schema}' union all select viewname as name, schemaname as schema, 'view' as type from pg_views - where schemaname = '{schema}' + where schemaname ilike '{schema}' """.format(schema=schema).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name, @@ -133,6 +133,10 @@ def list_relations(cls, profile, schema, model_name=None): database=profile.get('dbname'), schema=_schema, identifier=name, + quote_policy={ + 'schema': True, + 'identifier': True + }, type=type) for (name, _schema, type) in results] diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 376000e9d64..160a5b372b7 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -127,6 +127,10 @@ def list_relations(cls, profile, schema, model_name=None): database=profile.get('database'), schema=_schema, identifier=name, + quote_policy={ + 'schema': True, + 'identifier': True + }, type=relation_type_lookup.get(type)) for (name, _schema, type) in results] diff --git a/dbt/context/common.py b/dbt/context/common.py index 56ced6044fb..959ee79e98c 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -281,6 +281,18 @@ def get_this_relation(db_wrapper, profile, model): profile, model, table_name=table_name) +def create_relation(relation_type, quoting_config): + + class RelationWithContext(relation_type): + @classmethod + def create(cls, *args, **kwargs): + return relation_type.create(*args, + quote_policy=quoting_config, + **kwargs) + + return RelationWithContext + + def generate(model, project, flat_graph, provider=None): """ Not meant to be called directly. Call with either: @@ -310,7 +322,8 @@ def generate(model, project, flat_graph, provider=None): context = dbt.utils.merge(context, { "adapter": db_wrapper, "api": { - "Relation": adapter.Relation, + "Relation": create_relation(adapter.Relation, + project.get('quoting')), "Column": adapter.Column, }, "column": adapter.Column, diff --git a/dbt/context/parser.py b/dbt/context/parser.py index 3cce11d298d..ecd4052b538 100644 --- a/dbt/context/parser.py +++ b/dbt/context/parser.py @@ -18,7 +18,9 @@ def ref(*args): dbt.exceptions.ref_invalid_args(model, args) adapter = get_adapter(profile) - return adapter.Relation.create_from_node(profile, model) + return adapter.Relation.create_from_node( + profile, model, + quote_policy=project.get('quoting')) return ref diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index 5beb3ca4029..c78ff109b8a 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -57,7 +57,9 @@ def do_ref(*args): identifier=add_ephemeral_model_prefix( target_model_name)).quote(identifier=False) else: - return adapter.Relation.create_from_node(profile, target_model) + return adapter.Relation.create_from_node( + profile, target_model, + quote_policy=project.get('quoting')) return do_ref diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index e5dd0cba9ab..495119d6c41 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -25,6 +25,7 @@ Required('post-hook'): [hook_contract], Required('pre-hook'): [hook_contract], Required('vars'): dict, + Required('quoting'): dict, }, extra=ALLOW_EXTRA) parsed_node_contract = unparsed_node_contract.extend({ diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql index b98f6e3f95d..690d6f0bf07 100644 --- a/dbt/include/global_project/macros/materializations/table/table.sql +++ b/dbt/include/global_project/macros/materializations/table/table.sql @@ -6,9 +6,10 @@ {%- set existing_relations = adapter.list_relations(schema=schema) -%} {%- set old_relation = adapter.get_relation(relations_list=existing_relations, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} + {%- set target_relation = api.Relation.create(identifier=identifier, + schema=schema, type='table') -%} {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, - schema=schema, type='table') -%} + schema=schema, type='table') -%} {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} {%- set create_as_temporary = (exists_as_table and non_destructive_mode) -%} diff --git a/dbt/main.py b/dbt/main.py index 3282090156f..8b7ecd3cf84 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -175,6 +175,7 @@ def invoke_dbt(parsed): profile_to_load=parsed.profile, args=parsed ) + proj.log_warnings() proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") diff --git a/dbt/model.py b/dbt/model.py index 6c304dbe1a1..61bd61259ba 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -12,7 +12,7 @@ class SourceConfig(object): ConfigKeys = DBTConfigKeys AppendListFields = ['pre-hook', 'post-hook'] - ExtendDictFields = ['vars'] + ExtendDictFields = ['vars', 'quoting'] ClobberFields = [ 'schema', 'enabled', @@ -72,7 +72,8 @@ def config(self): active_config = self.load_config_from_active_project() if self.active_project['name'] == self.own_project['name']: - cfg = self._merge(defaults, active_config, self.in_model_config) + cfg = self._merge(defaults, active_config, + self.in_model_config) else: own_config = self.load_config_from_own_project() diff --git a/dbt/project.py b/dbt/project.py index 802f05226de..4d6aa50642f 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -91,6 +91,9 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None, if self.cfg.get('models') is None: self.cfg['models'] = {} + if self.cfg['models'].get('quoting') is None: + self.cfg['models']['quoting'] = {} + if self.cfg['models'].get('vars') is None: self.cfg['models']['vars'] = {} @@ -223,6 +226,21 @@ def validate(self): "Expected project configuration '{}' was not supplied" .format('.'.join(e.path)), self) + def log_warnings(self): + target_cfg = self.run_environment() + db_type = target_cfg.get('type') + + if db_type == 'snowflake' and self.cfg \ + .get('quoting', {}) \ + .get('identifier') is None: + logger.warn( + 'You are using Snowflake, but you did not specify a ' + 'quoting strategy for your identifiers. Quoting ' + 'behavior for Snowflake will change in a future release, ' + 'so it is recommended that you define this explicitly. ' + '\n\n' + 'For more information, see ADD LINK') + def hashed_name(self): if self.cfg.get("name", None) is None: return None diff --git a/dbt/utils.py b/dbt/utils.py index f34447d18ba..51bc1106c41 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -27,6 +27,7 @@ 'post-hook', 'vars', 'bind', + 'quoting', ] From a0dedfa0280e18a2104cf223ac50b9e67075cc7e Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Sun, 22 Apr 2018 15:43:26 -0400 Subject: [PATCH 41/61] fix default quoting config in project --- dbt/context/parser.py | 2 +- dbt/context/runtime.py | 2 +- dbt/project.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/context/parser.py b/dbt/context/parser.py index ecd4052b538..66bced09006 100644 --- a/dbt/context/parser.py +++ b/dbt/context/parser.py @@ -20,7 +20,7 @@ def ref(*args): adapter = get_adapter(profile) return adapter.Relation.create_from_node( profile, model, - quote_policy=project.get('quoting')) + quote_policy=project.get('quoting', {})) return ref diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index c78ff109b8a..f2248267659 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -59,7 +59,7 @@ def do_ref(*args): else: return adapter.Relation.create_from_node( profile, target_model, - quote_policy=project.get('quoting')) + quote_policy=project.get('quoting', {})) return do_ref diff --git a/dbt/project.py b/dbt/project.py index 4d6aa50642f..83481abd5ea 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -91,8 +91,8 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None, if self.cfg.get('models') is None: self.cfg['models'] = {} - if self.cfg['models'].get('quoting') is None: - self.cfg['models']['quoting'] = {} + if self.cfg.get('quoting') is None: + self.cfg['quoting'] = {} if self.cfg['models'].get('vars') is None: self.cfg['models']['vars'] = {} From 4a550e5ccac5314242d17dcca5ed50f421acaa54 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 23 Apr 2018 21:18:44 -0400 Subject: [PATCH 42/61] relations, fix another merge conflict --- dbt/adapters/default/impl.py | 4 ---- .../global_project/macros/materializations/seed/seed.sql | 6 ++++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 1c04b3cdc89..71cd92190fc 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -40,7 +40,6 @@ class DefaultAdapter(object): # just deprecated. going away in a future release "quote_schema_and_table", -<<<<<<< HEAD:dbt/adapters/default/impl.py # versions of adapter functions that take / return Relations "list_relations", @@ -51,10 +50,7 @@ class DefaultAdapter(object): "execute", "add_query", -======= - "execute", "convert_type" ->>>>>>> 5fefbbd21494f50952c03fa0025325b4aaab249f:dbt/adapters/default.py ] raw_functions = [ diff --git a/dbt/include/global_project/macros/materializations/seed/seed.sql b/dbt/include/global_project/macros/materializations/seed/seed.sql index 5aad5212c2a..57b2efe88c9 100644 --- a/dbt/include/global_project/macros/materializations/seed/seed.sql +++ b/dbt/include/global_project/macros/materializations/seed/seed.sql @@ -14,9 +14,10 @@ {% macro default__create_csv_table(model) %} {%- set agate_table = model['agate_table'] -%} {%- set column_override = model['config'].get('column_types', {}) -%} + {%- set target_relation = api.Relation.create_from_node(model) -%} {% set sql %} - create table {{ adapter.quote(model['schema']) }}.{{ adapter.quote(model['name']) }} ( + create table {{ target_relation }} ( {% for col_name in agate_table.column_names %} {% set inferred_type = adapter.convert_type(agate_table, loop.index0) %} {% set type = column_override.get(col_name, inferred_type) %} @@ -52,6 +53,7 @@ {% set cols_sql = ", ".join(agate_table.column_names) %} {% set bindings = [] %} + {% set target_relation = api.Relation.create_from_node(model) %} {% set statements = [] %} {% for chunk in agate_table.rows | batch(10000) %} @@ -62,7 +64,7 @@ {% endfor %} {% set sql %} - insert into {{ adapter.quote(model['schema']) }}.{{ adapter.quote(model['name']) }} ({{ cols_sql }}) values + insert into {{ target_relation }} ({{ cols_sql }}) values {% for row in chunk -%} ({%- for column in agate_table.column_names -%} %s From ba58e7e5d0e41f391e5af52365dad744a921d839 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 23 Apr 2018 23:06:39 -0400 Subject: [PATCH 43/61] seed + relations --- dbt/adapters/default/relation.py | 7 +- dbt/adapters/snowflake/relation.py | 17 ++--- dbt/context/common.py | 7 +- .../macros/materializations/seed/seed.sql | 46 +++++++------ .../models/compound_sort.sql | 2 +- .../001_simple_copy_test/models/disabled.sql | 2 +- .../models/incremental.sql | 2 +- .../models/interleaved_sort.sql | 2 +- .../models/materialized.sql | 2 +- .../models/view_model.sql | 2 +- .../001_simple_copy_test/test_simple_copy.py | 64 +++++++++++++++++-- test/integration/base.py | 48 ++++++++++---- 12 files changed, 137 insertions(+), 64 deletions(-) diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 17d380facbe..3006c39a387 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -118,7 +118,7 @@ def include(self, database=None, schema=None, identifier=None): return self.incorporate(include_policy=policy) - def render(self): + def render(self, use_table_name=True): parts = [] for k in self.PATH_ELEMENTS: @@ -128,7 +128,10 @@ def render(self): if path_part is None: continue elif k == 'identifier': - path_part = self.table + if use_table_name: + path_part = self.table + else: + path_part = self.identifier parts.append( self.quote_if( diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index 1db87909ead..d168ba67901 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -11,7 +11,7 @@ class SnowflakeRelation(DefaultRelation): 'quote_policy': { 'database': False, 'schema': False, - 'identifier': False, + 'identifier': True, }, 'include_policy': { 'database': False, @@ -61,17 +61,10 @@ def matches(self, database=None, schema=None, identifier=None): pass for k, v in search.items(): - # snowflake upcases unquoted identiifers. so, when - # comparing unquoted identifiers, use case insensitive - # matching. when comparing quoted identifiers, use case - # sensitive matching. - if self.should_quote(k): - if self.get_path_part(k) != v: - return False - - else: - if self.get_path_part(k) != v.upper(): - return False + # snowflake upcases unquoted identiifers. so, use + # case insensitive matching. + if self.get_path_part(k).upper() != v.upper(): + return False return True diff --git a/dbt/context/common.py b/dbt/context/common.py index 959ee79e98c..1fc25df4bbf 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -273,12 +273,13 @@ def _return(value): raise dbt.exceptions.MacroReturn(value) -def get_this_relation(db_wrapper, profile, model): +def get_this_relation(db_wrapper, project, profile, model): table_name = dbt.utils.model_immediate_name( model, dbt.flags.NON_DESTRUCTIVE) return db_wrapper.adapter.Relation.create_from_node( - profile, model, table_name=table_name) + profile, model, table_name=table_name, + quote_policy=project.get('quoting', {})) def create_relation(relation_type, quoting_config): @@ -349,7 +350,7 @@ def generate(model, project, flat_graph, provider=None): "fromjson": fromjson, "tojson": tojson, "target": target, - "this": get_this_relation(db_wrapper, profile, model), + "this": get_this_relation(db_wrapper, project, profile, model), "try_or_compiler_error": try_or_compiler_error(model) }) diff --git a/dbt/include/global_project/macros/materializations/seed/seed.sql b/dbt/include/global_project/macros/materializations/seed/seed.sql index 57b2efe88c9..de18b3de3de 100644 --- a/dbt/include/global_project/macros/materializations/seed/seed.sql +++ b/dbt/include/global_project/macros/materializations/seed/seed.sql @@ -3,8 +3,8 @@ {{ adapter_macro('create_csv_table', model) }} {%- endmacro %} -{% macro reset_csv_table(model, full_refresh, existing) -%} - {{ adapter_macro('reset_csv_table', model, full_refresh, existing) }} +{% macro reset_csv_table(model, full_refresh, old_relation) -%} + {{ adapter_macro('reset_csv_table', model, full_refresh, old_relation) }} {%- endmacro %} {% macro load_csv_rows(model) -%} @@ -14,15 +14,14 @@ {% macro default__create_csv_table(model) %} {%- set agate_table = model['agate_table'] -%} {%- set column_override = model['config'].get('column_types', {}) -%} - {%- set target_relation = api.Relation.create_from_node(model) -%} {% set sql %} - create table {{ target_relation }} ( - {% for col_name in agate_table.column_names %} - {% set inferred_type = adapter.convert_type(agate_table, loop.index0) %} - {% set type = column_override.get(col_name, inferred_type) %} - {{ col_name | string }} {{ type }} {% if not loop.last %}, {% endif %} - {% endfor %} + create table {{ this.render(False) }} ( + {%- for col_name in agate_table.column_names -%} + {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%} + {%- set type = column_override.get(col_name, inferred_type) -%} + {{ col_name | string }} {{ type }} {%- if not loop.last -%}, {%- endif -%} + {%- endfor -%} ) {% endset %} @@ -34,14 +33,14 @@ {% endmacro %} -{% macro default__reset_csv_table(model, full_refresh, existing) %} +{% macro default__reset_csv_table(model, full_refresh, old_relation) %} {% set sql = "" %} {% if full_refresh %} - {{ drop_if_exists(existing, model['schema'], model['name']) }} + {{ adapter.drop_relation(old_relation) }} {% set sql = create_csv_table(model) %} {% else %} - {{ adapter.truncate(model['schema'], model['name']) }} - {% set sql = "truncate table " ~ adapter.quote(model['schema']) ~ "." ~ adapter.quote(model['name']) %} + {{ adapter.truncate_relation(old_relation) }} + {% set sql = "truncate table " ~ old_relation %} {% endif %} {{ return(sql) }} @@ -53,7 +52,6 @@ {% set cols_sql = ", ".join(agate_table.column_names) %} {% set bindings = [] %} - {% set target_relation = api.Relation.create_from_node(model) %} {% set statements = [] %} {% for chunk in agate_table.rows | batch(10000) %} @@ -64,7 +62,7 @@ {% endfor %} {% set sql %} - insert into {{ target_relation }} ({{ cols_sql }}) values + insert into {{ this.render(False) }} ({{ cols_sql }}) values {% for row in chunk -%} ({%- for column in agate_table.column_names -%} %s @@ -90,8 +88,14 @@ {%- set identifier = model['name'] -%} {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - {%- set existing = adapter.query_for_existing(schema) -%} - {%- set existing_type = existing.get(identifier) -%} + {%- set existing_relations = adapter.list_relations(schema=schema) -%} + + {%- set old_relation = adapter.get_relation(relations_list=existing_relations, + schema=schema, identifier=identifier) -%} + + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + {%- set csv_table = model["agate_table"] -%} {{ run_hooks(pre_hooks, inside_transaction=False) }} @@ -101,12 +105,12 @@ -- build model {% set create_table_sql = "" %} - {% if existing_type and existing_type != 'table' %} + {% if exists_as_view %} {{ dbt.exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(identifier)) }} - {% elif existing_type is none%} - {% set create_table_sql = create_csv_table(model) %} + {% elif exists_as_table %} + {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation) %} {% else %} - {% set create_table_sql = reset_csv_table(model, full_refresh_mode, existing) %} + {% set create_table_sql = create_csv_table(model) %} {% endif %} {% set status = 'CREATE' if full_refresh_mode else 'INSERT' %} diff --git a/test/integration/001_simple_copy_test/models/compound_sort.sql b/test/integration/001_simple_copy_test/models/compound_sort.sql index 9ccc9374e3a..64b41ca7ebd 100644 --- a/test/integration/001_simple_copy_test/models/compound_sort.sql +++ b/test/integration/001_simple_copy_test/models/compound_sort.sql @@ -6,4 +6,4 @@ ) }} -select * from {{ target.schema }}.seed +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/disabled.sql b/test/integration/001_simple_copy_test/models/disabled.sql index 0f9d1d12c1c..1d10a0c8d41 100644 --- a/test/integration/001_simple_copy_test/models/disabled.sql +++ b/test/integration/001_simple_copy_test/models/disabled.sql @@ -5,4 +5,4 @@ ) }} -select * from {{ target.schema }}.seed +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/incremental.sql b/test/integration/001_simple_copy_test/models/incremental.sql index ec68f6c4824..72e0c872e6b 100644 --- a/test/integration/001_simple_copy_test/models/incremental.sql +++ b/test/integration/001_simple_copy_test/models/incremental.sql @@ -5,4 +5,4 @@ ) }} -select * from {{ target.schema }}.seed +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/interleaved_sort.sql b/test/integration/001_simple_copy_test/models/interleaved_sort.sql index 02e8c48812c..147370396e2 100644 --- a/test/integration/001_simple_copy_test/models/interleaved_sort.sql +++ b/test/integration/001_simple_copy_test/models/interleaved_sort.sql @@ -6,4 +6,4 @@ ) }} -select * from {{ target.schema }}.seed +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/materialized.sql b/test/integration/001_simple_copy_test/models/materialized.sql index 91387ccacda..d29676d672b 100644 --- a/test/integration/001_simple_copy_test/models/materialized.sql +++ b/test/integration/001_simple_copy_test/models/materialized.sql @@ -5,4 +5,4 @@ }} -- this is a unicode character: å -select * from {{ target.schema }}.seed +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/models/view_model.sql b/test/integration/001_simple_copy_test/models/view_model.sql index c7a2b6a6996..9838c5b839a 100644 --- a/test/integration/001_simple_copy_test/models/view_model.sql +++ b/test/integration/001_simple_copy_test/models/view_model.sql @@ -4,4 +4,4 @@ ) }} -select * from {{ target.schema }}.seed +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index e11e65ea07a..d4f05722f99 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -60,14 +60,66 @@ def test__snowflake__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view_model") - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") self.use_default_project({"data-paths": [self.dir("seed-update")]}) self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed","view_model") - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") + + @attr(type="snowflake") + def test__snowflake__simple_copy__quoting_on(self): + self.use_default_project({ + "data-paths": [self.dir("seed-initial")], + "quoting": {"identifier": True}, + }) + self.use_profile("snowflake") + + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual('"seed"', '"view_model"') + self.assertTablesEqual('"seed"', '"incremental"') + self.assertTablesEqual('"seed"', '"materialized"') + + self.use_default_project({ + "data-paths": [self.dir("seed-update")], + "quoting": {"identifier": True}, + }) + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual('"seed"', '"view_model"') + self.assertTablesEqual('"seed"', '"incremental"') + self.assertTablesEqual('"seed"', '"materialized"') + + @attr(type="snowflake") + def test__snowflake__simple_copy__quoting_off(self): + self.use_default_project({ + "data-paths": [self.dir("seed-initial")], + "quoting": {"identifier": False}, + }) + self.use_profile("snowflake") + + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual('"seed"', '"view_model"') + self.assertTablesEqual('"seed"', '"incremental"') + self.assertTablesEqual('"seed"', '"materialized"') + + self.use_default_project({ + "data-paths": [self.dir("seed-update")], + "quoting": {"identifier": False}, + }) + self.run_dbt(["seed"]) + self.run_dbt() + + self.assertTablesEqual('"seed"', '"view_model"') + self.assertTablesEqual('"seed"', '"incremental"') + self.assertTablesEqual('"seed"', '"materialized"') diff --git a/test/integration/base.py b/test/integration/base.py index 134b0a031cb..2c543d8a878 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -194,6 +194,9 @@ def use_default_project(self, overrides=None): 'version': '1.0', 'test-paths': [], 'source-paths': [self.models], + 'quoting': { + 'identifier': False, + }, 'profile': 'test', } @@ -276,8 +279,10 @@ def run_dbt(self, args=None, expect_pass=True): args = ["--strict"] + args logger.info("Invoking dbt with {}".format(args)) - res, success = dbt.handle_and_check(args) - self.assertEqual(success, expect_pass, "dbt exit state did not match expected") + res, success = dbt.handle_and_check(args) + self.assertEqual( + success, expect_pass, + "dbt exit state did not match expected") return res @@ -335,7 +340,8 @@ def get_table_columns(self, table, schema=None): and table_schema ilike '{}' order by column_name asc""" - result = self.run_sql(sql.format(table, schema), fetch='all') + result = self.run_sql(sql.format(table.replace('"', ''), schema), + fetch='all') return result @@ -381,15 +387,21 @@ def _assertTablesEqualSql(self, table_a_schema, table_a, table_b_schema, table_b return sql - def assertTablesEqual(self, table_a, table_b, table_a_schema=None, table_b_schema=None): - table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema - table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema + def assertTablesEqual(self, table_a, table_b, + table_a_schema=None, table_b_schema=None): + table_a_schema = self.unique_schema() \ + if table_a_schema is None else table_a_schema - self.assertTableColumnsEqual(table_a, table_b, table_a_schema, table_b_schema) - self.assertTableRowCountsEqual(table_a, table_b, table_a_schema, table_b_schema) + table_b_schema = self.unique_schema() \ + if table_b_schema is None else table_b_schema + self.assertTableColumnsEqual(table_a, table_b, + table_a_schema, table_b_schema) + self.assertTableRowCountsEqual(table_a, table_b, + table_a_schema, table_b_schema) - sql = self._assertTablesEqualSql(table_a_schema, table_a, table_b_schema, table_b) + sql = self._assertTablesEqualSql(table_a_schema, table_a, + table_b_schema, table_b) result = self.run_sql(sql, fetch='one') self.assertEquals( @@ -398,12 +410,20 @@ def assertTablesEqual(self, table_a, table_b, table_a_schema=None, table_b_schem sql ) - def assertTableRowCountsEqual(self, table_a, table_b, table_a_schema=None, table_b_schema=None): - table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema - table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema + def assertTableRowCountsEqual(self, table_a, table_b, + table_a_schema=None, table_b_schema=None): + table_a_schema = self.unique_schema() \ + if table_a_schema is None else table_a_schema + + table_b_schema = self.unique_schema() \ + if table_b_schema is None else table_b_schema - table_a_result = self.run_sql('SELECT COUNT(*) FROM {}.{}'.format(table_a_schema, table_a), fetch='one') - table_b_result = self.run_sql('SELECT COUNT(*) FROM {}.{}'.format(table_b_schema, table_b), fetch='one') + table_a_result = self.run_sql( + 'SELECT COUNT(*) FROM {}.{}' + .format(table_a_schema, table_a), fetch='one') + table_b_result = self.run_sql( + 'SELECT COUNT(*) FROM {}.{}' + .format(table_b_schema, table_b), fetch='one') self.assertEquals( table_a_result[0], From 2f54e525b24c330830aa81934e5ab81965626734 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 23 Apr 2018 23:20:03 -0400 Subject: [PATCH 44/61] fix unit tests --- test/unit/test_compiler.py | 1 + test/unit/test_parser.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index c47b2d699f3..9e2fe88eca1 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -38,6 +38,7 @@ def setUp(self): 'post-hook': [], 'pre-hook': [], 'vars': {}, + 'quoting': {}, 'column_types': {}, } diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 249cd840368..86b8fd2fe28 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -57,6 +57,7 @@ def setUp(self): 'post-hook': [], 'pre-hook': [], 'vars': {}, + 'quoting': {}, 'column_types': {}, } @@ -66,6 +67,7 @@ def setUp(self): 'post-hook': [], 'pre-hook': [], 'vars': {}, + 'quoting': {}, 'column_types': {}, } From 1c61341ea997a6c8997007f300b63ea83966669a Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Mon, 23 Apr 2018 23:38:54 -0400 Subject: [PATCH 45/61] add snowflake quoting off tests --- .../001_simple_copy_test/test_simple_copy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index d4f05722f99..bdb5478a83f 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -109,9 +109,9 @@ def test__snowflake__simple_copy__quoting_off(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual('"seed"', '"view_model"') - self.assertTablesEqual('"seed"', '"incremental"') - self.assertTablesEqual('"seed"', '"materialized"') + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") self.use_default_project({ "data-paths": [self.dir("seed-update")], @@ -120,6 +120,6 @@ def test__snowflake__simple_copy__quoting_off(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual('"seed"', '"view_model"') - self.assertTablesEqual('"seed"', '"incremental"') - self.assertTablesEqual('"seed"', '"materialized"') + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") From c7f83fd106afa64bfd6e1bd52572e6cc28d1167d Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 24 Apr 2018 09:49:54 -0400 Subject: [PATCH 46/61] fix bad merge --- dbt/adapters/default/impl.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 1c04b3cdc89..71cd92190fc 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -40,7 +40,6 @@ class DefaultAdapter(object): # just deprecated. going away in a future release "quote_schema_and_table", -<<<<<<< HEAD:dbt/adapters/default/impl.py # versions of adapter functions that take / return Relations "list_relations", @@ -51,10 +50,7 @@ class DefaultAdapter(object): "execute", "add_query", -======= - "execute", "convert_type" ->>>>>>> 5fefbbd21494f50952c03fa0025325b4aaab249f:dbt/adapters/default.py ] raw_functions = [ From 8694d3f6852d8ae572b331c69998747839383ffd Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 24 Apr 2018 10:21:04 -0400 Subject: [PATCH 47/61] fix for view --> table model switch --- .../global_project/macros/materializations/table/table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/dbt/include/global_project/macros/materializations/table/table.sql index b98f6e3f95d..909ca507abe 100644 --- a/dbt/include/global_project/macros/materializations/table/table.sql +++ b/dbt/include/global_project/macros/materializations/table/table.sql @@ -59,7 +59,7 @@ {% if non_destructive_mode -%} -- noop {%- else -%} - {{ drop_relation_if_exists(target_relation) }} + {{ drop_relation_if_exists(old_relation) }} {{ adapter.rename_relation(intermediate_relation, target_relation) }} {%- endif %} From 5bafcd605a34ef4caa301b9b4043ae27101bffc7 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 24 Apr 2018 11:22:58 -0400 Subject: [PATCH 48/61] fix exception code for seed onto view --- .../global_project/macros/materializations/seed/seed.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/seed/seed.sql b/dbt/include/global_project/macros/materializations/seed/seed.sql index de18b3de3de..f0a8ca86f95 100644 --- a/dbt/include/global_project/macros/materializations/seed/seed.sql +++ b/dbt/include/global_project/macros/materializations/seed/seed.sql @@ -106,7 +106,7 @@ -- build model {% set create_table_sql = "" %} {% if exists_as_view %} - {{ dbt.exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(identifier)) }} + {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }} {% elif exists_as_table %} {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation) %} {% else %} From ac62a4d1bdee3470364d474747fe4696c32eae92 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 24 Apr 2018 14:29:31 -0400 Subject: [PATCH 49/61] projects everywhere --- dbt/adapters/bigquery/impl.py | 66 +++++++++--------- dbt/adapters/default/impl.py | 67 ++++++++++--------- dbt/adapters/postgres/impl.py | 42 ++++++------ dbt/adapters/redshift/impl.py | 4 +- dbt/context/common.py | 16 +++-- dbt/main.py | 3 +- dbt/node_runners.py | 17 ++--- .../001_simple_copy_test/test_simple_copy.py | 10 +-- .../test_varchar_widening.py | 4 +- .../test_simple_reference.py | 12 ++-- .../test_simple_archive.py | 4 +- .../test_graph_selection.py | 16 ++--- .../test_schema_test_graph_selection.py | 8 +-- .../020_ephemeral_test/test_ephemeral.py | 4 +- .../021_concurrency_test/test_concurrency.py | 4 +- .../test_bigquery_date_partitioning.py | 1 - .../023_exit_codes_test/test_exit_codes.py | 14 ++-- .../integration/028_cli_vars/test_cli_vars.py | 6 +- test/integration/base.py | 28 +++++--- 19 files changed, 172 insertions(+), 154 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 7bd0c73691c..8b4b4c0fdd2 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -176,12 +176,12 @@ def close(cls, connection): return connection @classmethod - def list_relations(cls, profile, schema, model_name=None): + def list_relations(cls, profile, project, schema, model_name=None): connection = cls.get_connection(profile, model_name) credentials = connection.get('credentials', {}) client = connection.get('handle') - bigquery_dataset = cls.get_dataset(profile, schema, model_name) + bigquery_dataset = cls.get_dataset(profile, project, schema, model_name) all_tables = client.list_tables(bigquery_dataset) relation_types = { @@ -202,21 +202,21 @@ def list_relations(cls, profile, schema, model_name=None): for table in all_tables] @classmethod - def drop_relation(cls, profile, relation, model_name=None): + def drop_relation(cls, profile, project, relation, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, relation.schema, model_name) + dataset = cls.get_dataset(profile, project, relation.schema, model_name) relation_object = dataset.table(relation.identifier) client.delete_table(relation_object) @classmethod - def rename(cls, profile, schema, from_name, to_name, model_name=None): + def rename(cls, profile, project, schema, from_name, to_name, model_name=None): raise dbt.exceptions.NotImplementedException( '`rename` is not implemented for this adapter!') @classmethod - def rename_relation(cls, profile, from_relation, to_relation, + def rename_relation(cls, profile, project, from_relation, to_relation, model_name=None): raise dbt.exceptions.NotImplementedException( '`rename_relation` is not implemented for this adapter!') @@ -227,11 +227,11 @@ def get_timeout(cls, conn): return credentials.get('timeout_seconds', cls.QUERY_TIMEOUT) @classmethod - def materialize_as_view(cls, profile, dataset, model): + def materialize_as_view(cls, profile, project, dataset, model): model_name = model.get('name') model_sql = model.get('injected_sql') - conn = cls.get_connection(profile, model_name) + conn = cls.get_connection(profile, project, model_name) client = conn.get('handle') view_ref = dataset.table(model_name) @@ -262,12 +262,12 @@ def poll_until_job_completes(cls, job, timeout): raise job.exception() @classmethod - def make_date_partitioned_table(cls, profile, dataset_name, + def make_date_partitioned_table(cls, profile, project, dataset_name, identifier, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, dataset_name, identifier) + dataset = cls.get_dataset(profile, project, dataset_name, identifier) table_ref = dataset.table(identifier) table = google.cloud.bigquery.Table(table_ref) table.partitioning_type = 'DAY' @@ -275,7 +275,7 @@ def make_date_partitioned_table(cls, profile, dataset_name, return client.create_table(table) @classmethod - def materialize_as_table(cls, profile, dataset, model, model_sql, + def materialize_as_table(cls, profile, project, dataset, model, model_sql, decorator=None): model_name = model.get('name') @@ -302,7 +302,7 @@ def materialize_as_table(cls, profile, dataset, model, model_sql, return "CREATE TABLE" @classmethod - def execute_model(cls, profile, model, materialization, sql_override=None, + def execute_model(cls, profile, project, model, materialization, sql_override=None, decorator=None, model_name=None): if sql_override is None: @@ -315,12 +315,12 @@ def execute_model(cls, profile, model, materialization, sql_override=None, model_name = model.get('name') model_schema = model.get('schema') - dataset = cls.get_dataset(profile, model_schema, model_name) + dataset = cls.get_dataset(profile, project, model_schema, model_name) if materialization == 'view': - res = cls.materialize_as_view(profile, dataset, model) + res = cls.materialize_as_view(profile, project, dataset, model) elif materialization == 'table': - res = cls.materialize_as_table(profile, dataset, model, + res = cls.materialize_as_table(profile, project, dataset, model, sql_override, decorator) else: msg = "Invalid relation type: '{}'".format(materialization) @@ -369,18 +369,18 @@ def add_begin_query(cls, profile, name): '`add_begin_query` is not implemented for this adapter!') @classmethod - def create_schema(cls, profile, schema, model_name=None): + def create_schema(cls, profile, project, schema, model_name=None): logger.debug('Creating schema "%s".', schema) conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, schema, model_name) + dataset = cls.get_dataset(profile, project, schema, model_name) with cls.exception_handler(profile, 'create dataset', model_name): client.create_dataset(dataset) @classmethod - def drop_tables_in_schema(cls, profile, dataset): + def drop_tables_in_schema(cls, profile, project, dataset): conn = cls.get_connection(profile) client = conn.get('handle') @@ -388,22 +388,22 @@ def drop_tables_in_schema(cls, profile, dataset): client.delete_table(table.reference) @classmethod - def drop_schema(cls, profile, schema, model_name=None): + def drop_schema(cls, profile, project, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) - if not cls.check_schema_exists(profile, schema, model_name): + if not cls.check_schema_exists(profile, project, schema, model_name): return conn = cls.get_connection(profile) client = conn.get('handle') - dataset = cls.get_dataset(profile, schema, model_name) + dataset = cls.get_dataset(profile, project, schema, model_name) with cls.exception_handler(profile, 'drop dataset', model_name): - cls.drop_tables_in_schema(profile, dataset) + cls.drop_tables_in_schema(profile, project, dataset) client.delete_dataset(dataset) @classmethod - def get_existing_schemas(cls, profile, model_name=None): + def get_existing_schemas(cls, profile, project, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -412,7 +412,7 @@ def get_existing_schemas(cls, profile, model_name=None): return [ds.dataset_id for ds in all_datasets] @classmethod - def get_columns_in_table(cls, profile, schema_name, table_name, + def get_columns_in_table(cls, profile, project, schema_name, table_name, database=None, model_name=None): # BigQuery does not have databases -- the database parameter is here @@ -439,7 +439,7 @@ def get_columns_in_table(cls, profile, schema_name, table_name, return columns @classmethod - def check_schema_exists(cls, profile, schema, model_name=None): + def check_schema_exists(cls, profile, project, schema, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -448,7 +448,7 @@ def check_schema_exists(cls, profile, schema, model_name=None): return any([ds.dataset_id == schema for ds in all_datasets]) @classmethod - def get_dataset(cls, profile, dataset_name, model_name=None): + def get_dataset(cls, profile, project, dataset_name, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -479,13 +479,13 @@ def quote(cls, identifier): return '`{}`'.format(identifier) @classmethod - def quote_schema_and_table(cls, profile, schema, table, model_name=None): - return cls.render_relation(profile, + def quote_schema_and_table(cls, profile, project, schema, table, model_name=None): + return cls.render_relation(profile, project, cls.quote(schema), cls.quote(table)) @classmethod - def render_relation(cls, profile, schema, table): + def render_relation(cls, profile, project, schema, table): connection = cls.get_connection(profile) credentials = connection.get('credentials', {}) project = credentials.get('project') @@ -519,10 +519,10 @@ def _agate_to_schema(cls, agate_table, column_override): return bq_schema @classmethod - def load_dataframe(cls, profile, schema, table_name, agate_table, + def load_dataframe(cls, profile, project, schema, table_name, agate_table, column_override, model_name=None): bq_schema = cls._agate_to_schema(agate_table, column_override) - dataset = cls.get_dataset(profile, schema, None) + dataset = cls.get_dataset(profile, project, schema, None) table = dataset.table(table_name) conn = cls.get_connection(profile, None) client = conn.get('handle') @@ -539,7 +539,7 @@ def load_dataframe(cls, profile, schema, table_name, agate_table, cls.poll_until_job_completes(job, cls.get_timeout(conn)) @classmethod - def expand_target_column_types(cls, profile, temp_table, to_schema, - to_table, model_name=None): + def expand_target_column_types(cls, profile, project, temp_table, + to_schema, to_table, model_name=None): # This is a no-op on BigQuery pass diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 71cd92190fc..bbd51194d93 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -47,16 +47,18 @@ class DefaultAdapter(object): "drop_relation", "rename_relation", "truncate_relation", + ] + profile_functions = [ "execute", "add_query", - "convert_type" ] raw_functions = [ "get_status", "get_result_from_cursor", "quote", + "convert_type" ] Relation = DefaultRelation @@ -89,13 +91,13 @@ def get_status(cls, cursor): '`get_status` is not implemented for this adapter!') @classmethod - def alter_column_type(cls, profile, schema, table, column_name, + def alter_column_type(cls, profile, project, schema, table, column_name, new_column_type, model_name=None): raise dbt.exceptions.NotImplementedException( '`alter_column_type` is not implemented for this adapter!') @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): + def query_for_existing(cls, profile, project, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): schemas = [schemas] @@ -103,18 +105,18 @@ def query_for_existing(cls, profile, schemas, model_name=None): for schema in schemas: all_relations.extend( - cls.list_relations(profile, schema, model_name)) + cls.list_relations(profile, project, schema, model_name)) return {relation.identifier: relation.type for relation in all_relations} @classmethod - def get_existing_schemas(cls, profile, model_name=None): + def get_existing_schemas(cls, profile, project, model_name=None): raise dbt.exceptions.NotImplementedException( '`get_existing_schemas` is not implemented for this adapter!') @classmethod - def check_schema_exists(cls, profile, schema): + def check_schema_exists(cls, profile, project, schema): raise dbt.exceptions.NotImplementedException( '`check_schema_exists` is not implemented for this adapter!') @@ -139,17 +141,17 @@ def get_result_from_cursor(cls, cursor): return dbt.clients.agate_helper.table_from_data(data) @classmethod - def drop(cls, profile, schema, relation, relation_type, model_name=None): + def drop(cls, profile, project, schema, relation, relation_type, model_name=None): identifier = relation relation = cls.Relation.create( schema=schema, identifier=identifier, type=relation_type) - return cls.drop_relation(profile, relation, model_name) + return cls.drop_relation(profile, project, relation, model_name) @classmethod - def drop_relation(cls, profile, relation, model_name=None): + def drop_relation(cls, profile, project, relation, model_name=None): if relation.type is None: dbt.exceptions.raise_compiler_error( 'Tried to drop relation {}, but its type is null.' @@ -160,24 +162,24 @@ def drop_relation(cls, profile, relation, model_name=None): connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def truncate(cls, profile, schema, table, model_name=None): + def truncate(cls, profile, project, schema, table, model_name=None): relation = cls.Relation.create( schema=schema, identifier=table, type='table') - return cls.truncate_relation(profile, relation, model_name) + return cls.truncate_relation(profile, project, relation, model_name) @classmethod - def truncate_relation(cls, profile, relation, model_name=None): + def truncate_relation(cls, profile, project, relation, model_name=None): sql = 'truncate table {}'.format(relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def rename(cls, profile, schema, from_name, to_name, model_name=None): + def rename(cls, profile, project, schema, from_name, to_name, model_name=None): return cls.rename_relation( - profile, + profile, project, from_relation=cls.Relation.create( schema=schema, identifier=from_name), to_relation=cls.Relation.create( @@ -185,7 +187,7 @@ def rename(cls, profile, schema, from_name, to_name, model_name=None): model_name=model_name) @classmethod - def rename_relation(cls, profile, from_relation, + def rename_relation(cls, profile, project, from_relation, to_relation, model_name=None): sql = 'alter table {} rename to {}'.format( from_relation, to_relation.include(schema=False)) @@ -197,7 +199,7 @@ def is_cancelable(cls): return True @classmethod - def get_missing_columns(cls, profile, + def get_missing_columns(cls, profile, project, from_schema, from_table, to_schema, to_table, model_name=None): @@ -205,11 +207,11 @@ def get_missing_columns(cls, profile, missing from to_table""" from_columns = {col.name: col for col in cls.get_columns_in_table( - profile, from_schema, from_table, + profile, project, from_schema, from_table, model_name=model_name)} to_columns = {col.name: col for col in cls.get_columns_in_table( - profile, to_schema, to_table, + profile, project, to_schema, to_table, model_name=model_name)} missing_columns = set(from_columns.keys()) - set(to_columns.keys()) @@ -244,7 +246,7 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): return sql @classmethod - def get_columns_in_table(cls, profile, schema_name, table_name, + def get_columns_in_table(cls, profile, project, schema_name, table_name, database=None, model_name=None): sql = cls._get_columns_in_table_sql(schema_name, table_name, database) connection, cursor = cls.add_query( @@ -265,18 +267,18 @@ def _table_columns_to_dict(cls, columns): return {col.name: col for col in columns} @classmethod - def expand_target_column_types(cls, profile, + def expand_target_column_types(cls, profile, project, temp_table, to_schema, to_table, model_name=None): reference_columns = cls._table_columns_to_dict( cls.get_columns_in_table( - profile, None, temp_table, model_name=model_name)) + profile, project, None, temp_table, model_name=model_name)) target_columns = cls._table_columns_to_dict( cls.get_columns_in_table( - profile, to_schema, to_table, model_name=model_name)) + profile, project, to_schema, to_table, model_name=model_name)) for column_name, reference_column in reference_columns.items(): target_column = target_columns.get(column_name) @@ -291,19 +293,19 @@ def expand_target_column_types(cls, profile, to_schema, to_table) - cls.alter_column_type(profile, to_schema, to_table, + cls.alter_column_type(profile, project, to_schema, to_table, column_name, new_type, model_name) ### # RELATIONS ### @classmethod - def list_relations(cls, profile, schema, model_name=None): + def list_relations(cls, profile, project, schema, model_name=None): raise dbt.exceptions.NotImplementedException( '`list_relations` is not implemented for this adapter!') @classmethod - def get_relation(cls, profile, schema=None, relations_list=None, + def get_relation(cls, profile, project, schema=None, relations_list=None, model_name=None, **kwargs): if schema is None and relations_list is None: raise dbt.exceptions.RuntimeException( @@ -311,7 +313,7 @@ def get_relation(cls, profile, schema=None, relations_list=None, 'of relations to use') if relations_list is None: - relations_list = cls.list_relations(profile, schema, model_name) + relations_list = cls.list_relations(profile, project, schema, model_name) matches = [] @@ -346,7 +348,7 @@ def get_drop_schema_sql(cls, schema): # although some adapters may override them ### @classmethod - def get_default_schema(cls, profile): + def get_default_schema(cls, profile, project): return profile.get('schema') @classmethod @@ -663,7 +665,7 @@ def execute_all(cls, profile, sqls, model_name=None): return connection @classmethod - def create_schema(cls, profile, schema, model_name=None): + def create_schema(cls, profile, project, schema, model_name=None): logger.debug('Creating schema "%s".', schema) sql = cls.get_create_schema_sql(schema) res = cls.add_query(profile, sql, model_name) @@ -673,13 +675,13 @@ def create_schema(cls, profile, schema, model_name=None): return res @classmethod - def drop_schema(cls, profile, schema, model_name=None): + def drop_schema(cls, profile, project, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) sql = cls.get_drop_schema_sql(schema) return cls.add_query(profile, sql, model_name) @classmethod - def already_exists(cls, profile, schema, table, model_name=None): + def already_exists(cls, profile, project, schema, table, model_name=None): relation = cls.get_relation(profile, schema=schema, identifier=table) return relation is not None @@ -688,7 +690,8 @@ def quote(cls, identifier): return '"{}"'.format(identifier.replace('"', '""')) @classmethod - def quote_schema_and_table(cls, profile, schema, table, model_name=None): + def quote_schema_and_table(cls, profile, project, + schema, table, model_name=None): return '{}.{}'.format(cls.quote(schema), cls.quote(table)) @@ -723,7 +726,7 @@ def convert_time_type(cls, agate_table, col_idx): '`convert_time_type` is not implemented for this adapter!') @classmethod - def convert_type(cls, profiel, agate_table, col_idx, model_name): + def convert_type(cls, agate_table, col_idx): return cls.convert_agate_type(agate_table, col_idx) @classmethod diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 67fdb80e979..64258e2dde4 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -7,7 +7,6 @@ import dbt.exceptions import agate -from dbt.utils import chunks from dbt.logger import GLOBAL_LOGGER as logger @@ -17,8 +16,6 @@ class PostgresAdapter(dbt.adapters.default.DefaultAdapter): @contextmanager def exception_handler(cls, profile, sql, model_name=None, connection_name=None): - connection = cls.get_connection(profile, connection_name) - try: yield @@ -86,7 +83,24 @@ def open_connection(cls, connection): return result @classmethod - def alter_column_type(cls, profile, schema, table, column_name, + def cancel_connection(cls, profile, connection): + connection_name = connection.get('name') + pid = connection.get('handle').get_backend_pid() + + sql = "select pg_terminate_backend({})".format(pid) + + logger.debug("Cancelling query '{}' ({})".format(connection_name, pid)) + + _, cursor = cls.add_query(profile, sql, 'master') + res = cursor.fetchone() + + logger.debug("Cancel query '{}': {}".format(connection_name, res)) + + # DATABASE INSPECTION FUNCTIONS + # These require the profile AND project, as they need to know + # database-specific configs at the project level. + @classmethod + def alter_column_type(cls, profile, project, schema, table, column_name, new_column_type, model_name=None): """ 1. Create a new column (w/ temp name and correct type) @@ -115,7 +129,7 @@ def alter_column_type(cls, profile, schema, table, column_name, return connection, cursor @classmethod - def list_relations(cls, profile, schema, model_name=None): + def list_relations(cls, profile, project, schema, model_name=None): sql = """ select tablename as name, schemaname as schema, 'table' as type from pg_tables where schemaname ilike '{schema}' @@ -141,7 +155,7 @@ def list_relations(cls, profile, schema, model_name=None): for (name, _schema, type) in results] @classmethod - def get_existing_schemas(cls, profile, model_name=None): + def get_existing_schemas(cls, profile, project, model_name=None): sql = "select distinct nspname from pg_namespace" connection, cursor = cls.add_query(profile, sql, model_name, @@ -151,7 +165,7 @@ def get_existing_schemas(cls, profile, model_name=None): return [row[0] for row in results] @classmethod - def check_schema_exists(cls, profile, schema, model_name=None): + def check_schema_exists(cls, profile, project, schema, model_name=None): sql = """ select count(*) from pg_namespace where nspname = '{schema}' """.format(schema=schema).strip() # noqa @@ -162,20 +176,6 @@ def check_schema_exists(cls, profile, schema, model_name=None): return results[0] > 0 - @classmethod - def cancel_connection(cls, profile, connection): - connection_name = connection.get('name') - pid = connection.get('handle').get_backend_pid() - - sql = "select pg_terminate_backend({})".format(pid) - - logger.debug("Cancelling query '{}' ({})".format(connection_name, pid)) - - _, cursor = cls.add_query(profile, sql, 'master') - res = cursor.fetchone() - - logger.debug("Cancel query '{}': {}".format(connection_name, res)) - @classmethod def convert_text_type(cls, agate_table, col_idx): return "text" diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 310632773fc..311e09d3923 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -91,7 +91,7 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): return sql @classmethod - def drop_relation(cls, profile, relation, model_name=None): + def drop_relation(cls, profile, project, relation, model_name=None): """ In Redshift, DROP TABLE ... CASCADE should not be used inside a transaction. Redshift doesn't prevent the CASCADE @@ -121,7 +121,7 @@ def drop_relation(cls, profile, relation, model_name=None): cls.begin(profile, connection.get('name')) to_return = super(PostgresAdapter, cls).drop_relation( - profile, relation, model_name) + profile, project, relation, model_name) cls.commit(profile, connection) cls.begin(profile, connection.get('name')) diff --git a/dbt/context/common.py b/dbt/context/common.py index 1fc25df4bbf..fe56666fbc1 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -30,10 +30,11 @@ class DatabaseWrapper(object): functions. """ - def __init__(self, model, adapter, profile): + def __init__(self, model, adapter, profile, project): self.model = model self.adapter = adapter self.profile = profile + self.project = project self.Relation = adapter.Relation # Fun with metaprogramming @@ -43,16 +44,21 @@ def __init__(self, model, adapter, profile): for context_function in self.adapter.context_functions: setattr(self, context_function, - self.wrap_with_profile_and_model_name(context_function)) + self.wrap(context_function, (self.profile, self.project,))) + + for profile_function in self.adapter.profile_functions: + setattr(self, + profile_function, + self.wrap(profile_function, (self.profile,))) for raw_function in self.adapter.raw_functions: setattr(self, raw_function, getattr(self.adapter, raw_function)) - def wrap_with_profile_and_model_name(self, fn): + def wrap(self, fn, arg_prefix): def wrapped(*args, **kwargs): - args = (self.profile,) + args + args = arg_prefix + args kwargs['model_name'] = self.model.get('name') return getattr(self.adapter, fn)(*args, **kwargs) @@ -318,7 +324,7 @@ def generate(model, project, flat_graph, provider=None): pre_hooks = model.get('config', {}).get('pre-hook') post_hooks = model.get('config', {}).get('post-hook') - db_wrapper = DatabaseWrapper(model, adapter, profile) + db_wrapper = DatabaseWrapper(model, adapter, profile, project) context = dbt.utils.merge(context, { "adapter": db_wrapper, diff --git a/dbt/main.py b/dbt/main.py index 8b7ecd3cf84..29388240bab 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -175,7 +175,6 @@ def invoke_dbt(parsed): profile_to_load=parsed.profile, args=parsed ) - proj.log_warnings() proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") @@ -233,6 +232,8 @@ def invoke_dbt(parsed): return None + proj.log_warnings() + flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False) arg_drop_existing = getattr(proj.args, 'drop_existing', False) diff --git a/dbt/node_runners.py b/dbt/node_runners.py index 3e415fe4ed3..a26ab388034 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -250,17 +250,18 @@ def _node_context(cls, adapter, project, node): def call_get_columns_in_table(schema_name, table_name): return adapter.get_columns_in_table( - profile, schema_name, table_name, model_name=node.get('name')) + profile, project, schema_name, + table_name, model_name=node.get('name')) def call_get_missing_columns(from_schema, from_table, to_schema, to_table): return adapter.get_missing_columns( - profile, from_schema, from_table, + profile, project, from_schema, from_table, to_schema, to_table, node.get('name')) def call_already_exists(schema, table): return adapter.already_exists( - profile, schema, table, node.get('name')) + profile, project, schema, table, node.get('name')) return { "run_started_at": dbt.tracking.active_user.run_started_at, @@ -274,9 +275,9 @@ def call_already_exists(schema, table): def create_schemas(cls, project, adapter, flat_graph): profile = project.run_environment() required_schemas = cls.get_model_schemas(flat_graph) - existing_schemas = set(adapter.get_existing_schemas(profile)) + existing_schemas = set(adapter.get_existing_schemas(profile, project)) for schema in (required_schemas - existing_schemas): - adapter.create_schema(profile, schema) + adapter.create_schema(profile, project, schema) class ModelRunner(CompileRunner): @@ -345,12 +346,12 @@ def create_schemas(cls, project, adapter, flat_graph): # is the one defined in the profile. Create this schema if it # does not exist, otherwise subsequent queries will fail. Generally, # dbt expects that this schema will exist anyway. - required_schemas.add(adapter.get_default_schema(profile)) + required_schemas.add(adapter.get_default_schema(profile, project)) - existing_schemas = set(adapter.get_existing_schemas(profile)) + existing_schemas = set(adapter.get_existing_schemas(profile, project)) for schema in (required_schemas - existing_schemas): - adapter.create_schema(profile, schema) + adapter.create_schema(profile, project, schema) @classmethod def before_run(cls, project, adapter, flat_graph): diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index bdb5478a83f..4dc7dcf4118 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -21,8 +21,8 @@ def models(self): @attr(type="postgres") def test__postgres__simple_copy(self): - self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.use_profile("postgres") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.run_dbt(["seed"]) self.run_dbt() @@ -41,8 +41,8 @@ def test__postgres__simple_copy(self): @attr(type="postgres") def test__postgres__dbt_doesnt_run_empty_models(self): - self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.use_profile("postgres") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.run_dbt(["seed"]) self.run_dbt() @@ -54,8 +54,8 @@ def test__postgres__dbt_doesnt_run_empty_models(self): @attr(type="snowflake") def test__snowflake__simple_copy(self): - self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.use_profile("snowflake") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) self.run_dbt(["seed"]) self.run_dbt() @@ -74,11 +74,11 @@ def test__snowflake__simple_copy(self): @attr(type="snowflake") def test__snowflake__simple_copy__quoting_on(self): + self.use_profile("snowflake") self.use_default_project({ "data-paths": [self.dir("seed-initial")], "quoting": {"identifier": True}, }) - self.use_profile("snowflake") self.run_dbt(["seed"]) self.run_dbt() @@ -100,11 +100,11 @@ def test__snowflake__simple_copy__quoting_on(self): @attr(type="snowflake") def test__snowflake__simple_copy__quoting_off(self): + self.use_profile("snowflake") self.use_default_project({ "data-paths": [self.dir("seed-initial")], "quoting": {"identifier": False}, }) - self.use_profile("snowflake") self.run_dbt(["seed"]) self.run_dbt() diff --git a/test/integration/002_varchar_widening_test/test_varchar_widening.py b/test/integration/002_varchar_widening_test/test_varchar_widening.py index fcf9a6b7d83..34ddb89624a 100644 --- a/test/integration/002_varchar_widening_test/test_varchar_widening.py +++ b/test/integration/002_varchar_widening_test/test_varchar_widening.py @@ -16,8 +16,8 @@ def models(self): @attr(type='postgres') def test__postgres__varchar_widening(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/002_varchar_widening_test/seed.sql") self.run_dbt() @@ -34,8 +34,8 @@ def test__postgres__varchar_widening(self): @attr(type='snowflake') def test__snowflake__varchar_widening(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/002_varchar_widening_test/seed.sql") self.run_dbt() diff --git a/test/integration/003_simple_reference_test/test_simple_reference.py b/test/integration/003_simple_reference_test/test_simple_reference.py index bdbef08807f..b4efdcc58bc 100644 --- a/test/integration/003_simple_reference_test/test_simple_reference.py +++ b/test/integration/003_simple_reference_test/test_simple_reference.py @@ -16,8 +16,8 @@ def models(self): @attr(type='postgres') def test__postgres__simple_reference(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") self.run_dbt() @@ -50,8 +50,8 @@ def test__postgres__simple_reference(self): @attr(type='snowflake') def test__snowflake__simple_reference(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") self.run_dbt() @@ -84,8 +84,8 @@ def test__snowflake__simple_reference(self): @attr(type='postgres') def test__postgres__simple_reference_with_models(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy, ephemeral_copy, and their dependents @@ -100,8 +100,8 @@ def test__postgres__simple_reference_with_models(self): @attr(type='postgres') def test__postgres__simple_reference_with_models_and_children(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy, ephemeral_copy, and their dependents @@ -136,8 +136,8 @@ def test__postgres__simple_reference_with_models_and_children(self): @attr(type='snowflake') def test__snowflake__simple_reference_with_models(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy & ephemeral_copy @@ -152,8 +152,8 @@ def test__snowflake__simple_reference_with_models(self): @attr(type='snowflake') def test__snowflake__simple_reference_with_models_and_children(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") # Run materialized_copy, ephemeral_copy, and their dependents diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py index 51b7bce4d74..308d85af5bc 100644 --- a/test/integration/004_simple_archive_test/test_simple_archive.py +++ b/test/integration/004_simple_archive_test/test_simple_archive.py @@ -35,8 +35,8 @@ def project_config(self): @attr(type='postgres') def test__postgres__simple_archive(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/004_simple_archive_test/seed.sql") self.run_dbt(["archive"]) @@ -52,8 +52,8 @@ def test__postgres__simple_archive(self): @attr(type='snowflake') def test__snowflake__simple_archive(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/004_simple_archive_test/seed.sql") self.run_dbt(["archive"]) diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index 86aef1abc87..02444031b10 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -13,8 +13,8 @@ def models(self): @attr(type='postgres') def test__postgres__specific_model(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users']) @@ -27,8 +27,8 @@ def test__postgres__specific_model(self): @attr(type='snowflake') def test__snowflake__specific_model(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users']) @@ -42,8 +42,8 @@ def test__snowflake__specific_model(self): @attr(type='postgres') def test__postgres__specific_model_and_children(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users+']) @@ -56,8 +56,8 @@ def test__postgres__specific_model_and_children(self): @attr(type='snowflake') def test__snowflake__specific_model_and_children(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', 'users+']) @@ -71,8 +71,8 @@ def test__snowflake__specific_model_and_children(self): @attr(type='postgres') def test__postgres__specific_model_and_parents(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup']) @@ -85,8 +85,8 @@ def test__postgres__specific_model_and_parents(self): @attr(type='snowflake') def test__snowflake__specific_model_and_parents(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup']) @@ -100,8 +100,8 @@ def test__snowflake__specific_model_and_parents(self): @attr(type='postgres') def test__postgres__specific_model_with_exclusion(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup', '--exclude', 'users_rollup']) @@ -114,8 +114,8 @@ def test__postgres__specific_model_with_exclusion(self): @attr(type='snowflake') def test__snowflake__specific_model_with_exclusion(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(['run', '--models', '+users_rollup', '--exclude', 'users_rollup']) diff --git a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py index d01e490123f..c12a856c188 100644 --- a/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_schema_test_graph_selection.py @@ -23,14 +23,10 @@ def project_config(self): ] } - def setUp(self): - DBTIntegrationTest.setUp(self) - - self.use_default_project() - self.project = read_project('dbt_project.yml') - def run_schema_and_assert(self, include, exclude, expected_tests): self.use_profile('postgres') + self.use_default_project() + self.project = read_project('dbt_project.yml') self.run_sql_file("test/integration/007_graph_selection_tests/seed.sql") self.run_dbt(["deps"]) diff --git a/test/integration/020_ephemeral_test/test_ephemeral.py b/test/integration/020_ephemeral_test/test_ephemeral.py index 06f4f5c5894..682ad1eb767 100644 --- a/test/integration/020_ephemeral_test/test_ephemeral.py +++ b/test/integration/020_ephemeral_test/test_ephemeral.py @@ -16,8 +16,8 @@ def models(self): @attr(type='postgres') def test__postgres(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/020_ephemeral_test/seed.sql") result = self.run_dbt() @@ -27,8 +27,8 @@ def test__postgres(self): @attr(type='snowflake') def test__snowflake(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/020_ephemeral_test/seed.sql") self.run_dbt() diff --git a/test/integration/021_concurrency_test/test_concurrency.py b/test/integration/021_concurrency_test/test_concurrency.py index afa4d46a6db..be42d46b34b 100644 --- a/test/integration/021_concurrency_test/test_concurrency.py +++ b/test/integration/021_concurrency_test/test_concurrency.py @@ -17,8 +17,8 @@ def models(self): @attr(type='postgres') def test__postgres__concurrency(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_sql_file("test/integration/021_concurrency_test/seed.sql") self.run_dbt(expect_pass=False) @@ -43,8 +43,8 @@ def test__postgres__concurrency(self): @attr(type='snowflake') def test__snowflake__concurrency(self): - self.use_default_project() self.use_profile('snowflake') + self.use_default_project() self.run_sql_file("test/integration/021_concurrency_test/seed.sql") self.run_dbt(expect_pass=False) diff --git a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py b/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py index 3277c70f490..c4c946fca6e 100644 --- a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py +++ b/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py @@ -30,4 +30,3 @@ def test__bigquery_date_partitioning(self): self.assertFalse(result.skipped) # status = # of failing rows self.assertEqual(result.status, 0) - diff --git a/test/integration/023_exit_codes_test/test_exit_codes.py b/test/integration/023_exit_codes_test/test_exit_codes.py index 01af65ce2ec..3831c3298a8 100644 --- a/test/integration/023_exit_codes_test/test_exit_codes.py +++ b/test/integration/023_exit_codes_test/test_exit_codes.py @@ -35,24 +35,24 @@ def project_config(self): @attr(type='postgres') def test_exit_code_run_succeed(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'good']) self.assertTrue(success) self.assertTableDoesExist('good') @attr(type='postgres') def test__exit_code_run_fail(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'bad']) self.assertFalse(success) self.assertTableDoesNotExist('bad') @attr(type='postgres') def test___schema_test_pass(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'good']) self.assertTrue(success) _, success = self.run_dbt_and_check(['test', '--model', 'good']) @@ -60,8 +60,8 @@ def test___schema_test_pass(self): @attr(type='postgres') def test___schema_test_fail(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'dupe']) self.assertTrue(success) _, success = self.run_dbt_and_check(['test', '--model', 'dupe']) @@ -69,15 +69,15 @@ def test___schema_test_fail(self): @attr(type='postgres') def test___compile(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['compile']) self.assertTrue(success) @attr(type='postgres') def test___archive_pass(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_dbt_and_check(['run', '--model', 'good']) _, success = self.run_dbt_and_check(['archive']) @@ -115,8 +115,8 @@ def project_config(self): @attr(type='postgres') def test___archive_fail(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() _, success = self.run_dbt_and_check(['run', '--model', 'good']) self.assertTrue(success) diff --git a/test/integration/028_cli_vars/test_cli_vars.py b/test/integration/028_cli_vars/test_cli_vars.py index 347a20e4dac..001f9061df5 100644 --- a/test/integration/028_cli_vars/test_cli_vars.py +++ b/test/integration/028_cli_vars/test_cli_vars.py @@ -14,8 +14,8 @@ def models(self): @attr(type='postgres') def test__cli_vars_longform(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() cli_vars = { "variable_1": "abc", @@ -39,16 +39,16 @@ def models(self): @attr(type='postgres') def test__cli_vars_shorthand(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_dbt(["run", "--vars", "simple: abc"]) self.run_dbt(["test"]) @attr(type='postgres') def test__cli_vars_longer(self): - self.use_default_project() self.use_profile('postgres') + self.use_default_project() self.run_dbt(["run", "--vars", "{simple: abc, unused: def}"]) self.run_dbt(["test"]) diff --git a/test/integration/base.py b/test/integration/base.py index 2c543d8a878..3e992f9fc86 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -6,6 +6,7 @@ import json from dbt.adapters.factory import get_adapter +from dbt.project import Project from dbt.logger import GLOBAL_LOGGER as logger @@ -169,6 +170,8 @@ def setUp(self): profile = profile_config.get('test').get('outputs').get(target) + project = Project(project_config, profile_config, DBT_CONFIG_DIR) + adapter = get_adapter(profile) # it's important to use a different connection handle here so @@ -177,12 +180,14 @@ def setUp(self): connection = adapter.acquire_connection(profile, '__test') self.handle = connection.get('handle') self.adapter_type = profile.get('type') - self.profile = profile + self._profile = profile + self._profile_config = profile_config + self.project = project if self.adapter_type == 'bigquery': schema_name = self.unique_schema() - adapter.drop_schema(profile, schema_name, '__test') - adapter.create_schema(profile, schema_name, '__test') + adapter.drop_schema(profile, project, schema_name, '__test') + adapter.create_schema(profile, project, schema_name, '__test') else: self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) self.run_sql('CREATE SCHEMA {}'.format(self.unique_schema())) @@ -205,6 +210,9 @@ def use_default_project(self, overrides=None): project_config.update(self.project_config) project_config.update(overrides or {}) + project = Project(project_config, self._profile_config, DBT_CONFIG_DIR) + self.project = project + with open("dbt_project.yml", 'w') as f: yaml.safe_dump(project_config, f, default_flow_style=True) @@ -229,11 +237,14 @@ def use_profile(self, adapter_type): connection = adapter.acquire_connection(profile, '__test') self.handle = connection.get('handle') self.adapter_type = profile.get('type') - self.profile = profile + self._profile_config = profile_config + self._profile = profile if self.adapter_type == 'bigquery': - adapter.drop_schema(profile, self.unique_schema(), '__test') - adapter.create_schema(profile, self.unique_schema(), '__test') + adapter.drop_schema(profile, self.project, + self.unique_schema(), '__test') + adapter.create_schema(profile, self.project, + self.unique_schema(), '__test') else: self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) self.run_sql('CREATE SCHEMA {}'.format(self.unique_schema())) @@ -249,10 +260,11 @@ def tearDown(self): except: os.rename("dbt_modules", "dbt_modules-{}".format(time.time())) - adapter = get_adapter(self.profile) + adapter = get_adapter(self._profile) if self.adapter_type == 'bigquery': - adapter.drop_schema(self.profile, self.unique_schema(), '__test') + adapter.drop_schema(self._profile, self.project, + self.unique_schema(), '__test') else: self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE' .format(self.unique_schema())) From 8ee8e766fbc5280f654b640a1a1fd0ec4272193e Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 24 Apr 2018 16:34:12 -0400 Subject: [PATCH 50/61] test fixes --- dbt/adapters/default/impl.py | 40 ++++++++++--- dbt/adapters/default/relation.py | 15 ++++- dbt/adapters/snowflake/impl.py | 59 ++++++------------- dbt/exceptions.py | 16 ++++- .../test_seed_type_override.py | 4 +- 5 files changed, 79 insertions(+), 55 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index bbd51194d93..2b12e358cdf 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -305,25 +305,41 @@ def list_relations(cls, profile, project, schema, model_name=None): '`list_relations` is not implemented for this adapter!') @classmethod - def get_relation(cls, profile, project, schema=None, relations_list=None, - model_name=None, **kwargs): + def _make_match_kwargs(cls, project, schema, identifier): + if identifier is not None and \ + project.cfg.get('quoting', {}).get('identifier') is False: + identifier = identifier.lower() + + if schema is not None and \ + project.cfg.get('quoting', {}).get('schema') is False: + schema = schema.lower() + + return filter_null_values({'identifier': identifier, + 'schema': schema}) + + @classmethod + def get_relation(cls, profile, project, schema=None, identifier=None, + relations_list=None, model_name=None): if schema is None and relations_list is None: raise dbt.exceptions.RuntimeException( 'get_relation needs either a schema to query, or a list ' 'of relations to use') if relations_list is None: - relations_list = cls.list_relations(profile, project, schema, model_name) + relations_list = cls.list_relations( + profile, project, schema, model_name) matches = [] + search = cls._make_match_kwargs(project, schema, identifier) + for relation in relations_list: - if relation.matches(schema=schema, **kwargs): + if relation.matches(**search): matches.append(relation) if len(matches) > 1: dbt.exceptions.get_relation_returned_multiple_results( - kwargs, matches) + {'identifier': identifier, 'schema': schema}, matches) elif matches: return matches[0] @@ -334,12 +350,18 @@ def get_relation(cls, profile, project, schema=None, relations_list=None, # SANE ANSI SQL DEFAULTS ### @classmethod - def get_create_schema_sql(cls, schema): + def get_create_schema_sql(cls, project, schema): + if project.cfg.get('quoting', {}).get('schema'): + schema = cls.quote(schema) + return ('create schema if not exists {schema}' .format(schema=schema)) @classmethod - def get_drop_schema_sql(cls, schema): + def get_drop_schema_sql(cls, project, schema): + if project.cfg.get('quoting', {}).get('schema'): + schema = cls.quote(schema) + return ('drop schema if exists {schema} cascade' .format(schema=schema)) @@ -667,7 +689,7 @@ def execute_all(cls, profile, sqls, model_name=None): @classmethod def create_schema(cls, profile, project, schema, model_name=None): logger.debug('Creating schema "%s".', schema) - sql = cls.get_create_schema_sql(schema) + sql = cls.get_create_schema_sql(project, schema) res = cls.add_query(profile, sql, model_name) cls.commit_if_has_connection(profile, model_name) @@ -677,7 +699,7 @@ def create_schema(cls, profile, project, schema, model_name=None): @classmethod def drop_schema(cls, profile, project, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) - sql = cls.get_drop_schema_sql(schema) + sql = cls.get_drop_schema_sql(project, schema) return cls.add_query(profile, sql, model_name) @classmethod diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 3006c39a387..3bd354de69f 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -1,6 +1,8 @@ from dbt.api import APIObject from dbt.utils import filter_null_values +import dbt.exceptions + class DefaultRelation(APIObject): @@ -85,11 +87,20 @@ def matches(self, database=None, schema=None, identifier=None): # nothing was passed in pass + exact_match = True + approximate_match = True + for k, v in search.items(): if self.get_path_part(k) != v: - return False + exact_match = False + + if self.get_path_part(k).lower() != v.lower(): + approximate_match = False + + if approximate_match and not exact_match: + dbt.exceptions.approximate_relation_match(search, self) - return True + return exact_match def get_path_part(self, part): return self.path.get(part) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 160a5b372b7..6804209467e 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -14,6 +14,7 @@ from dbt.adapters.postgres import PostgresAdapter from dbt.adapters.snowflake.relation import SnowflakeRelation from dbt.logger import GLOBAL_LOGGER as logger +from dbt.utils import filter_null_values class SnowflakeAdapter(PostgresAdapter): @@ -105,7 +106,7 @@ def open_connection(cls, connection): return result @classmethod - def list_relations(cls, profile, schema, model_name=None): + def list_relations(cls, profile, project, schema, model_name=None): sql = """ select table_name as name, table_schema as schema, table_type as type @@ -135,34 +136,7 @@ def list_relations(cls, profile, schema, model_name=None): for (name, _schema, type) in results] @classmethod - def query_for_existing(cls, profile, schemas, model_name=None): - if not isinstance(schemas, (list, tuple)): - schemas = [schemas] - - schemas = ["upper('{}')".format(schema) for schema in schemas] - schema_list = ",".join(schemas) - - sql = """ - select table_name as name, table_type as type - from information_schema.tables - where upper(table_schema) in ({schema_list}) - """.format(schema_list=schema_list).strip() # noqa - - _, cursor = cls.add_query(profile, sql, model_name, auto_begin=False) - results = cursor.fetchall() - - relation_type_lookup = { - 'BASE TABLE': 'table', - 'VIEW': 'view' - } - - existing = [(name, relation_type_lookup.get(relation_type)) - for (name, relation_type) in results] - - return dict(existing) - - @classmethod - def rename_relation(cls, profile, from_relation, + def rename_relation(cls, profile, project, from_relation, to_relation, model_name=None): sql = 'alter table {} rename to {}'.format( from_relation, to_relation) @@ -174,17 +148,7 @@ def add_begin_query(cls, profile, name): return cls.add_query(profile, 'BEGIN', name, auto_begin=False) @classmethod - def create_schema(cls, profile, schema, model_name=None): - logger.debug('Creating schema "%s".', schema) - sql = cls.get_create_schema_sql(schema) - res = cls.add_query(profile, sql, model_name) - - cls.commit_if_has_connection(profile, model_name) - - return res - - @classmethod - def get_existing_schemas(cls, profile, model_name=None): + def get_existing_schemas(cls, profile, project, model_name=None): sql = "select distinct schema_name from information_schema.schemata" connection, cursor = cls.add_query(profile, sql, model_name, @@ -194,7 +158,7 @@ def get_existing_schemas(cls, profile, model_name=None): return [row[0] for row in results] @classmethod - def check_schema_exists(cls, profile, schema, model_name=None): + def check_schema_exists(cls, profile, project, schema, model_name=None): sql = """ select count(*) from information_schema.schemata @@ -254,6 +218,19 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True, return connection, cursor + @classmethod + def _make_match_kwargs(cls, project, schema, identifier): + if identifier is not None and \ + project.cfg.get('quoting', {}).get('identifier') is False: + identifier = identifier.upper() + + if schema is not None and \ + project.cfg.get('quoting', {}).get('schema') is False: + schema = schema.upper() + + return filter_null_values({'identifier': identifier, + 'schema': schema}) + @classmethod def cancel_connection(cls, profile, connection): handle = connection['handle'] diff --git a/dbt/exceptions.py b/dbt/exceptions.py index fa35eb1e4ee..48814c59434 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -305,7 +305,7 @@ def raise_dep_not_found(node, node_description, required_pkg): '`dbt deps`.'.format(node_description, required_pkg), node=node) -def get_relation_returned_multiple_results(kwargs, matches): +def multiple_matching_relations(kwargs, matches): raise_compiler_error( 'get_relation returned more than one relation with the given args. ' 'Please specify a database or schema to narrow down the result set.' @@ -313,6 +313,20 @@ def get_relation_returned_multiple_results(kwargs, matches): .format(kwargs, matches)) +def approximate_relation_match(search, relation): + raise_compiler_error( + 'When searching for a relation, dbt found an approximate match. ' + 'Instead of guessing which relation to use, dbt will abort. ' + 'Please delete the approximate match, or rename it to match the ' + 'identifier dbt is trying to edit.' + '\nSearch: {}\nRelation: {}' + .format(search, relation)) + + +def get_relation_returned_multiple_results(kwargs, matches): + multiple_matching_relations(kwargs, matches) + + def raise_duplicate_resource_name(node_1, node_2): duped_name = node_1['name'] diff --git a/test/integration/005_simple_seed_test/test_seed_type_override.py b/test/integration/005_simple_seed_test/test_seed_type_override.py index 658cd9d2065..98f26598693 100644 --- a/test/integration/005_simple_seed_test/test_seed_type_override.py +++ b/test/integration/005_simple_seed_test/test_seed_type_override.py @@ -1,6 +1,7 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest + class TestSimpleSeedColumnOverride(DBTIntegrationTest): @property @@ -23,6 +24,7 @@ def project_config(self): } } + class TestSimpleSeedColumnOverridePostgres(TestSimpleSeedColumnOverride): @property def models(self): @@ -83,5 +85,3 @@ def profile_config(self): def test_simple_seed_with_column_override_bq(self): self.run_dbt(["seed"]) self.run_dbt(["test"]) - - From dd707c88ca24bee5fbc025b3747796a7fc0eeb00 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 24 Apr 2018 17:02:00 -0400 Subject: [PATCH 51/61] switch test, fix schema creation / dropping --- dbt/adapters/default/impl.py | 11 +++++---- dbt/adapters/default/relation.py | 4 +++- dbt/adapters/snowflake/impl.py | 5 ++-- dbt/adapters/snowflake/relation.py | 23 ++----------------- dbt/exceptions.py | 20 ++++++++-------- .../001_simple_copy_test/test_simple_copy.py | 16 +++++++++++++ 6 files changed, 39 insertions(+), 40 deletions(-) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 2b12e358cdf..262ec76d612 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -141,7 +141,8 @@ def get_result_from_cursor(cls, cursor): return dbt.clients.agate_helper.table_from_data(data) @classmethod - def drop(cls, profile, project, schema, relation, relation_type, model_name=None): + def drop(cls, profile, project, schema, + relation, relation_type, model_name=None): identifier = relation relation = cls.Relation.create( schema=schema, @@ -177,7 +178,8 @@ def truncate_relation(cls, profile, project, relation, model_name=None): connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def rename(cls, profile, project, schema, from_name, to_name, model_name=None): + def rename(cls, profile, project, schema, + from_name, to_name, model_name=None): return cls.rename_relation( profile, project, from_relation=cls.Relation.create( @@ -221,7 +223,6 @@ def get_missing_columns(cls, profile, project, @classmethod def _get_columns_in_table_sql(cls, schema_name, table_name, database): - schema_filter = '1=1' if schema_name is not None: schema_filter = "table_schema = '{}'".format(schema_name) @@ -351,7 +352,7 @@ def get_relation(cls, profile, project, schema=None, identifier=None, ### @classmethod def get_create_schema_sql(cls, project, schema): - if project.cfg.get('quoting', {}).get('schema'): + if project.cfg.get('quoting', {}).get('schema', True): schema = cls.quote(schema) return ('create schema if not exists {schema}' @@ -359,7 +360,7 @@ def get_create_schema_sql(cls, project, schema): @classmethod def get_drop_schema_sql(cls, project, schema): - if project.cfg.get('quoting', {}).get('schema'): + if project.cfg.get('quoting', {}).get('schema', True): schema = cls.quote(schema) return ('drop schema if exists {schema} cascade' diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index 3bd354de69f..a3a5c561050 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -98,7 +98,9 @@ def matches(self, database=None, schema=None, identifier=None): approximate_match = False if approximate_match and not exact_match: - dbt.exceptions.approximate_relation_match(search, self) + target = self.create( + database=database, schema=schema, identifier=identifier) + dbt.exceptions.approximate_relation_match(target, self) return exact_match diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 6804209467e..206ad2e3887 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -221,11 +221,11 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True, @classmethod def _make_match_kwargs(cls, project, schema, identifier): if identifier is not None and \ - project.cfg.get('quoting', {}).get('identifier') is False: + project.cfg.get('quoting', {}).get('identifier', True) is False: identifier = identifier.upper() if schema is not None and \ - project.cfg.get('quoting', {}).get('schema') is False: + project.cfg.get('quoting', {}).get('schema', True) is False: schema = schema.upper() return filter_null_values({'identifier': identifier, @@ -249,7 +249,6 @@ def cancel_connection(cls, profile, connection): @classmethod def _get_columns_in_table_sql(cls, schema_name, table_name, database): - schema_filter = '1=1' if schema_name is not None: schema_filter = "table_schema ilike '{}'".format(schema_name) diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index d168ba67901..54667a054d4 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -9,8 +9,8 @@ class SnowflakeRelation(DefaultRelation): }, 'quote_character': '"', 'quote_policy': { - 'database': False, - 'schema': False, + 'database': True, + 'schema': True, 'identifier': True, }, 'include_policy': { @@ -49,24 +49,5 @@ def create_from_node(cls, profile, node, **kwargs): identifier=node.get('name'), **kwargs) - def matches(self, database=None, schema=None, identifier=None): - search = filter_null_values({ - 'database': database, - 'schema': schema, - 'identifier': identifier - }) - - if not search: - # nothing was passed in - pass - - for k, v in search.items(): - # snowflake upcases unquoted identiifers. so, use - # case insensitive matching. - if self.get_path_part(k).upper() != v.upper(): - return False - - return True - def get_path_part(self, part): return self.path.get(part) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 48814c59434..807dbd29eae 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -313,20 +313,20 @@ def multiple_matching_relations(kwargs, matches): .format(kwargs, matches)) -def approximate_relation_match(search, relation): - raise_compiler_error( - 'When searching for a relation, dbt found an approximate match. ' - 'Instead of guessing which relation to use, dbt will abort. ' - 'Please delete the approximate match, or rename it to match the ' - 'identifier dbt is trying to edit.' - '\nSearch: {}\nRelation: {}' - .format(search, relation)) - - def get_relation_returned_multiple_results(kwargs, matches): multiple_matching_relations(kwargs, matches) +def approximate_relation_match(target, relation): + raise_compiler_error( + 'When searching for a relation, dbt found an approximate match. ' + 'Instead of guessing \nwhich relation to use, dbt will exit. ' + 'Please delete {relation}, or rename it to be less ambiguous.' + '\nSearched for: {target}\nFound: {relation}' + .format(target=target, + relation=relation)) + + def raise_duplicate_resource_name(node_1, node_2): duped_name = node_1['name'] diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index 4dc7dcf4118..e48cd667969 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -123,3 +123,19 @@ def test__snowflake__simple_copy__quoting_off(self): self.assertTablesEqual("seed", "view_model") self.assertTablesEqual("seed", "incremental") self.assertTablesEqual("seed", "materialized") + + @attr(type="snowflake") + def test__snowflake__seed__quoting_switch(self): + self.use_profile("snowflake") + self.use_default_project({ + "data-paths": [self.dir("seed-initial")], + "quoting": {"identifier": False}, + }) + + self.run_dbt(["seed"]) + + self.use_default_project({ + "data-paths": [self.dir("seed-update")], + "quoting": {"identifier": True}, + }) + self.run_dbt(["seed"], expect_pass=False) From e269c25762ebe2a7d3c906b704d0bc152c0da487 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 24 Apr 2018 17:49:48 -0400 Subject: [PATCH 52/61] quote tests properly --- dbt/adapters/snowflake/relation.py | 1 - test/integration/base.py | 35 ++++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index 54667a054d4..65b0c83b6e3 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -1,5 +1,4 @@ from dbt.adapters.default.relation import DefaultRelation -from dbt.utils import filter_null_values class SnowflakeRelation(DefaultRelation): diff --git a/test/integration/base.py b/test/integration/base.py index 3e992f9fc86..02778ea6c06 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -180,6 +180,7 @@ def setUp(self): connection = adapter.acquire_connection(profile, '__test') self.handle = connection.get('handle') self.adapter_type = profile.get('type') + self.adapter = adapter self._profile = profile self._profile_config = profile_config self.project = project @@ -199,9 +200,6 @@ def use_default_project(self, overrides=None): 'version': '1.0', 'test-paths': [], 'source-paths': [self.models], - 'quoting': { - 'identifier': False, - }, 'profile': 'test', } @@ -232,6 +230,8 @@ def use_profile(self, adapter_type): profile = profile_config.get('test').get('outputs').get('default2') adapter = get_adapter(profile) + self.adapter = adapter + # it's important to use a different connection handle here so # we don't look into an incomplete transaction connection = adapter.acquire_connection(profile, '__test') @@ -266,8 +266,11 @@ def tearDown(self): adapter.drop_schema(self._profile, self.project, self.unique_schema(), '__test') else: + if self.project.cfg.get('quoting', {}).get('schema', True): + schema = self.adapter.quote(self.unique_schema()) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE' - .format(self.unique_schema())) + .format(schema)) self.handle.close() # hack for BQ -- TODO @@ -382,6 +385,14 @@ def _assertTablesEqualSql(self, table_a_schema, table_a, table_b_schema, table_b else: columns_csv = ", ".join(['{}'.format(record[0]) for record in columns]) + if self.project.cfg.get('quoting', {}).get('identifier', True): + table_a = self.adapter.quote(table_a) + table_b = self.adapter.quote(table_b) + + if self.project.cfg.get('quoting', {}).get('schema', True): + table_a_schema = self.adapter.quote(table_a_schema) + table_b_schema = self.adapter.quote(table_b_schema) + sql = """ SELECT COUNT(*) FROM ( (SELECT {columns} FROM {table_a_schema}.{table_a} EXCEPT @@ -430,6 +441,14 @@ def assertTableRowCountsEqual(self, table_a, table_b, table_b_schema = self.unique_schema() \ if table_b_schema is None else table_b_schema + if self.project.cfg.get('quoting', {}).get('identifier', True): + table_a = self.adapter.quote(table_a) + table_b = self.adapter.quote(table_b) + + if self.project.cfg.get('quoting', {}).get('schema', True): + table_a_schema = self.adapter.quote(table_a_schema) + table_b_schema = self.adapter.quote(table_b_schema) + table_a_result = self.run_sql( 'SELECT COUNT(*) FROM {}.{}' .format(table_a_schema, table_a), fetch='one') @@ -468,6 +487,14 @@ def assertTableColumnsEqual(self, table_a, table_b, table_a_schema=None, table_b table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema + if self.project.cfg.get('quoting', {}).get('identifier', True): + table_a = self.adapter.quote(table_a) + table_b = self.adapter.quote(table_b) + + if self.project.cfg.get('quoting', {}).get('schema', True): + table_a_schema = self.adapter.quote(table_a_schema) + table_b_schema = self.adapter.quote(table_b_schema) + table_a_result = self.get_table_columns(table_a, table_a_schema) table_b_result = self.get_table_columns(table_b, table_b_schema) From 8cc3696ae1ef6c9fe57a558c70b75ba4fcb25afb Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 25 Apr 2018 11:23:37 -0400 Subject: [PATCH 53/61] spruce up snowflake warning, add link --- dbt/links.py | 2 ++ dbt/project.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 dbt/links.py diff --git a/dbt/links.py b/dbt/links.py new file mode 100644 index 00000000000..3be2c275d9e --- /dev/null +++ b/dbt/links.py @@ -0,0 +1,2 @@ + +SnowflakeQuotingDocs = 'https://docs.getdbt.com/v0.10/docs/configuring-quoting' diff --git a/dbt/project.py b/dbt/project.py index 83481abd5ea..a16e38e49ef 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -12,6 +12,8 @@ import dbt.compat import dbt.context.common import dbt.clients.system +import dbt.ui.printer +import dbt.links from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -233,13 +235,15 @@ def log_warnings(self): if db_type == 'snowflake' and self.cfg \ .get('quoting', {}) \ .get('identifier') is None: - logger.warn( + msg = dbt.ui.printer.yellow( 'You are using Snowflake, but you did not specify a ' - 'quoting strategy for your identifiers. Quoting ' + 'quoting strategy for your identifiers.\nQuoting ' 'behavior for Snowflake will change in a future release, ' - 'so it is recommended that you define this explicitly. ' - '\n\n' - 'For more information, see ADD LINK') + 'so it is recommended that you define this explicitly.\n\n' + 'For more information, see: {}\n' + ) + + logger.warn(msg.format(dbt.links.SnowflakeQuotingDocs)) def hashed_name(self): if self.cfg.get("name", None) is None: From a661512772fe9bd3680eb9934bedf5ab19d2dc9d Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 25 Apr 2018 18:22:16 -0400 Subject: [PATCH 54/61] fix snowflake tests --- dbt/adapters/bigquery/impl.py | 65 +++++++------- dbt/adapters/default/impl.py | 85 ++++++++++--------- dbt/adapters/postgres/impl.py | 13 +-- dbt/adapters/snowflake/impl.py | 14 +-- dbt/compilation.py | 2 +- dbt/context/common.py | 46 +++++++--- dbt/context/parser.py | 13 +-- dbt/context/runtime.py | 17 ++-- dbt/exceptions.py | 2 +- .../materializations/archive/archive.sql | 32 +++---- dbt/node_runners.py | 3 +- .../001_simple_copy_test/test_simple_copy.py | 24 +++--- .../test_varchar_widening.py | 8 +- .../test_simple_reference.py | 56 ++++++------ .../test_simple_archive.py | 11 ++- .../test_graph_selection.py | 12 +-- .../020_ephemeral_test/test_ephemeral.py | 7 +- .../021_concurrency_test/test_concurrency.py | 16 ++-- test/integration/base.py | 69 ++++++--------- 19 files changed, 252 insertions(+), 243 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 8b4b4c0fdd2..de62e412535 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -176,12 +176,12 @@ def close(cls, connection): return connection @classmethod - def list_relations(cls, profile, project, schema, model_name=None): + def list_relations(cls, profile, project_cfg, schema, model_name=None): connection = cls.get_connection(profile, model_name) credentials = connection.get('credentials', {}) client = connection.get('handle') - bigquery_dataset = cls.get_dataset(profile, project, schema, model_name) + bigquery_dataset = cls.get_dataset(profile, project_cfg, schema, model_name) all_tables = client.list_tables(bigquery_dataset) relation_types = { @@ -202,21 +202,21 @@ def list_relations(cls, profile, project, schema, model_name=None): for table in all_tables] @classmethod - def drop_relation(cls, profile, project, relation, model_name=None): + def drop_relation(cls, profile, project_cfg, relation, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, project, relation.schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, relation.schema, model_name) relation_object = dataset.table(relation.identifier) client.delete_table(relation_object) @classmethod - def rename(cls, profile, project, schema, from_name, to_name, model_name=None): + def rename(cls, profile, project_cfg, schema, from_name, to_name, model_name=None): raise dbt.exceptions.NotImplementedException( '`rename` is not implemented for this adapter!') @classmethod - def rename_relation(cls, profile, project, from_relation, to_relation, + def rename_relation(cls, profile, project_cfg, from_relation, to_relation, model_name=None): raise dbt.exceptions.NotImplementedException( '`rename_relation` is not implemented for this adapter!') @@ -227,11 +227,11 @@ def get_timeout(cls, conn): return credentials.get('timeout_seconds', cls.QUERY_TIMEOUT) @classmethod - def materialize_as_view(cls, profile, project, dataset, model): + def materialize_as_view(cls, profile, project_cfg, dataset, model): model_name = model.get('name') model_sql = model.get('injected_sql') - conn = cls.get_connection(profile, project, model_name) + conn = cls.get_connection(profile, project_cfg, model_name) client = conn.get('handle') view_ref = dataset.table(model_name) @@ -262,12 +262,12 @@ def poll_until_job_completes(cls, job, timeout): raise job.exception() @classmethod - def make_date_partitioned_table(cls, profile, project, dataset_name, + def make_date_partitioned_table(cls, profile, project_cfg, dataset_name, identifier, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, project, dataset_name, identifier) + dataset = cls.get_dataset(profile, project_cfg, dataset_name, identifier) table_ref = dataset.table(identifier) table = google.cloud.bigquery.Table(table_ref) table.partitioning_type = 'DAY' @@ -275,7 +275,7 @@ def make_date_partitioned_table(cls, profile, project, dataset_name, return client.create_table(table) @classmethod - def materialize_as_table(cls, profile, project, dataset, model, model_sql, + def materialize_as_table(cls, profile, project_cfg, dataset, model, model_sql, decorator=None): model_name = model.get('name') @@ -302,7 +302,7 @@ def materialize_as_table(cls, profile, project, dataset, model, model_sql, return "CREATE TABLE" @classmethod - def execute_model(cls, profile, project, model, materialization, sql_override=None, + def execute_model(cls, profile, project_cfg, model, materialization, sql_override=None, decorator=None, model_name=None): if sql_override is None: @@ -315,12 +315,12 @@ def execute_model(cls, profile, project, model, materialization, sql_override=No model_name = model.get('name') model_schema = model.get('schema') - dataset = cls.get_dataset(profile, project, model_schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, model_schema, model_name) if materialization == 'view': - res = cls.materialize_as_view(profile, project, dataset, model) + res = cls.materialize_as_view(profile, project_cfg, dataset, model) elif materialization == 'table': - res = cls.materialize_as_table(profile, project, dataset, model, + res = cls.materialize_as_table(profile, project_cfg, dataset, model, sql_override, decorator) else: msg = "Invalid relation type: '{}'".format(materialization) @@ -369,18 +369,18 @@ def add_begin_query(cls, profile, name): '`add_begin_query` is not implemented for this adapter!') @classmethod - def create_schema(cls, profile, project, schema, model_name=None): + def create_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Creating schema "%s".', schema) conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, project, schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, schema, model_name) with cls.exception_handler(profile, 'create dataset', model_name): client.create_dataset(dataset) @classmethod - def drop_tables_in_schema(cls, profile, project, dataset): + def drop_tables_in_schema(cls, profile, project_cfg, dataset): conn = cls.get_connection(profile) client = conn.get('handle') @@ -388,22 +388,22 @@ def drop_tables_in_schema(cls, profile, project, dataset): client.delete_table(table.reference) @classmethod - def drop_schema(cls, profile, project, schema, model_name=None): + def drop_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) - if not cls.check_schema_exists(profile, project, schema, model_name): + if not cls.check_schema_exists(profile, project_cfg, schema, model_name): return conn = cls.get_connection(profile) client = conn.get('handle') - dataset = cls.get_dataset(profile, project, schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, schema, model_name) with cls.exception_handler(profile, 'drop dataset', model_name): - cls.drop_tables_in_schema(profile, project, dataset) + cls.drop_tables_in_schema(profile, project_cfg, dataset) client.delete_dataset(dataset) @classmethod - def get_existing_schemas(cls, profile, project, model_name=None): + def get_existing_schemas(cls, profile, project_cfg, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -412,7 +412,7 @@ def get_existing_schemas(cls, profile, project, model_name=None): return [ds.dataset_id for ds in all_datasets] @classmethod - def get_columns_in_table(cls, profile, project, schema_name, table_name, + def get_columns_in_table(cls, profile, project_cfg, schema_name, table_name, database=None, model_name=None): # BigQuery does not have databases -- the database parameter is here @@ -439,7 +439,7 @@ def get_columns_in_table(cls, profile, project, schema_name, table_name, return columns @classmethod - def check_schema_exists(cls, profile, project, schema, model_name=None): + def check_schema_exists(cls, profile, project_cfg, schema, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -448,7 +448,7 @@ def check_schema_exists(cls, profile, project, schema, model_name=None): return any([ds.dataset_id == schema for ds in all_datasets]) @classmethod - def get_dataset(cls, profile, project, dataset_name, model_name=None): + def get_dataset(cls, profile, project_cfg, dataset_name, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -479,13 +479,13 @@ def quote(cls, identifier): return '`{}`'.format(identifier) @classmethod - def quote_schema_and_table(cls, profile, project, schema, table, model_name=None): - return cls.render_relation(profile, project, + def quote_schema_and_table(cls, profile, project_cfg, schema, table, model_name=None): + return cls.render_relation(profile, project_cfg, cls.quote(schema), cls.quote(table)) @classmethod - def render_relation(cls, profile, project, schema, table): + def render_relation(cls, profile, project_cfg, schema, table): connection = cls.get_connection(profile) credentials = connection.get('credentials', {}) project = credentials.get('project') @@ -519,10 +519,11 @@ def _agate_to_schema(cls, agate_table, column_override): return bq_schema @classmethod - def load_dataframe(cls, profile, project, schema, table_name, agate_table, + def load_dataframe(cls, profile, project_cfg, schema, + table_name, agate_table, column_override, model_name=None): bq_schema = cls._agate_to_schema(agate_table, column_override) - dataset = cls.get_dataset(profile, project, schema, None) + dataset = cls.get_dataset(profile, project_cfg, schema, None) table = dataset.table(table_name) conn = cls.get_connection(profile, None) client = conn.get('handle') @@ -539,7 +540,7 @@ def load_dataframe(cls, profile, project, schema, table_name, agate_table, cls.poll_until_job_completes(job, cls.get_timeout(conn)) @classmethod - def expand_target_column_types(cls, profile, project, temp_table, + def expand_target_column_types(cls, profile, project_cfg, temp_table, to_schema, to_table, model_name=None): # This is a no-op on BigQuery pass diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 262ec76d612..9bad2c47152 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -91,13 +91,13 @@ def get_status(cls, cursor): '`get_status` is not implemented for this adapter!') @classmethod - def alter_column_type(cls, profile, project, schema, table, column_name, + def alter_column_type(cls, profile, project_cfg, schema, table, column_name, new_column_type, model_name=None): raise dbt.exceptions.NotImplementedException( '`alter_column_type` is not implemented for this adapter!') @classmethod - def query_for_existing(cls, profile, project, schemas, model_name=None): + def query_for_existing(cls, profile, project_cfg, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): schemas = [schemas] @@ -105,23 +105,23 @@ def query_for_existing(cls, profile, project, schemas, model_name=None): for schema in schemas: all_relations.extend( - cls.list_relations(profile, project, schema, model_name)) + cls.list_relations(profile, project_cfg, schema, model_name)) return {relation.identifier: relation.type for relation in all_relations} @classmethod - def get_existing_schemas(cls, profile, project, model_name=None): + def get_existing_schemas(cls, profile, project_cfg, model_name=None): raise dbt.exceptions.NotImplementedException( '`get_existing_schemas` is not implemented for this adapter!') @classmethod - def check_schema_exists(cls, profile, project, schema): + def check_schema_exists(cls, profile, project_cfg, schema): raise dbt.exceptions.NotImplementedException( '`check_schema_exists` is not implemented for this adapter!') @classmethod - def cancel_connection(cls, project, connection): + def cancel_connection(cls, project_cfg, connection): raise dbt.exceptions.NotImplementedException( '`cancel_connection` is not implemented for this adapter!') @@ -141,7 +141,7 @@ def get_result_from_cursor(cls, cursor): return dbt.clients.agate_helper.table_from_data(data) @classmethod - def drop(cls, profile, project, schema, + def drop(cls, profile, project_cfg, schema, relation, relation_type, model_name=None): identifier = relation relation = cls.Relation.create( @@ -149,10 +149,10 @@ def drop(cls, profile, project, schema, identifier=identifier, type=relation_type) - return cls.drop_relation(profile, project, relation, model_name) + return cls.drop_relation(profile, project_cfg, relation, model_name) @classmethod - def drop_relation(cls, profile, project, relation, model_name=None): + def drop_relation(cls, profile, project_cfg, relation, model_name=None): if relation.type is None: dbt.exceptions.raise_compiler_error( 'Tried to drop relation {}, but its type is null.' @@ -163,25 +163,25 @@ def drop_relation(cls, profile, project, relation, model_name=None): connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def truncate(cls, profile, project, schema, table, model_name=None): + def truncate(cls, profile, project_cfg, schema, table, model_name=None): relation = cls.Relation.create( schema=schema, identifier=table, type='table') - return cls.truncate_relation(profile, project, relation, model_name) + return cls.truncate_relation(profile, project_cfg, relation, model_name) @classmethod - def truncate_relation(cls, profile, project, relation, model_name=None): + def truncate_relation(cls, profile, project_cfg, relation, model_name=None): sql = 'truncate table {}'.format(relation) connection, cursor = cls.add_query(profile, sql, model_name) @classmethod - def rename(cls, profile, project, schema, + def rename(cls, profile, project_cfg, schema, from_name, to_name, model_name=None): return cls.rename_relation( - profile, project, + profile, project_cfg, from_relation=cls.Relation.create( schema=schema, identifier=from_name), to_relation=cls.Relation.create( @@ -189,7 +189,7 @@ def rename(cls, profile, project, schema, model_name=model_name) @classmethod - def rename_relation(cls, profile, project, from_relation, + def rename_relation(cls, profile, project_cfg, from_relation, to_relation, model_name=None): sql = 'alter table {} rename to {}'.format( from_relation, to_relation.include(schema=False)) @@ -201,7 +201,7 @@ def is_cancelable(cls): return True @classmethod - def get_missing_columns(cls, profile, project, + def get_missing_columns(cls, profile, project_cfg, from_schema, from_table, to_schema, to_table, model_name=None): @@ -209,11 +209,11 @@ def get_missing_columns(cls, profile, project, missing from to_table""" from_columns = {col.name: col for col in cls.get_columns_in_table( - profile, project, from_schema, from_table, + profile, project_cfg, from_schema, from_table, model_name=model_name)} to_columns = {col.name: col for col in cls.get_columns_in_table( - profile, project, to_schema, to_table, + profile, project_cfg, to_schema, to_table, model_name=model_name)} missing_columns = set(from_columns.keys()) - set(to_columns.keys()) @@ -247,7 +247,7 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): return sql @classmethod - def get_columns_in_table(cls, profile, project, schema_name, table_name, + def get_columns_in_table(cls, profile, project_cfg, schema_name, table_name, database=None, model_name=None): sql = cls._get_columns_in_table_sql(schema_name, table_name, database) connection, cursor = cls.add_query( @@ -268,18 +268,18 @@ def _table_columns_to_dict(cls, columns): return {col.name: col for col in columns} @classmethod - def expand_target_column_types(cls, profile, project, + def expand_target_column_types(cls, profile, project_cfg, temp_table, to_schema, to_table, model_name=None): reference_columns = cls._table_columns_to_dict( cls.get_columns_in_table( - profile, project, None, temp_table, model_name=model_name)) + profile, project_cfg, None, temp_table, model_name=model_name)) target_columns = cls._table_columns_to_dict( cls.get_columns_in_table( - profile, project, to_schema, to_table, model_name=model_name)) + profile, project_cfg, to_schema, to_table, model_name=model_name)) for column_name, reference_column in reference_columns.items(): target_column = target_columns.get(column_name) @@ -294,32 +294,32 @@ def expand_target_column_types(cls, profile, project, to_schema, to_table) - cls.alter_column_type(profile, project, to_schema, to_table, + cls.alter_column_type(profile, project_cfg, to_schema, to_table, column_name, new_type, model_name) ### # RELATIONS ### @classmethod - def list_relations(cls, profile, project, schema, model_name=None): + def list_relations(cls, profile, project_cfg, schema, model_name=None): raise dbt.exceptions.NotImplementedException( '`list_relations` is not implemented for this adapter!') @classmethod - def _make_match_kwargs(cls, project, schema, identifier): + def _make_match_kwargs(cls, project_cfg, schema, identifier): if identifier is not None and \ - project.cfg.get('quoting', {}).get('identifier') is False: + project_cfg.get('quoting', {}).get('identifier') is False: identifier = identifier.lower() if schema is not None and \ - project.cfg.get('quoting', {}).get('schema') is False: + project_cfg.get('quoting', {}).get('schema') is False: schema = schema.lower() return filter_null_values({'identifier': identifier, 'schema': schema}) @classmethod - def get_relation(cls, profile, project, schema=None, identifier=None, + def get_relation(cls, profile, project_cfg, schema=None, identifier=None, relations_list=None, model_name=None): if schema is None and relations_list is None: raise dbt.exceptions.RuntimeException( @@ -328,11 +328,11 @@ def get_relation(cls, profile, project, schema=None, identifier=None, if relations_list is None: relations_list = cls.list_relations( - profile, project, schema, model_name) + profile, project_cfg, schema, model_name) matches = [] - search = cls._make_match_kwargs(project, schema, identifier) + search = cls._make_match_kwargs(project_cfg, schema, identifier) for relation in relations_list: if relation.matches(**search): @@ -351,16 +351,16 @@ def get_relation(cls, profile, project, schema=None, identifier=None, # SANE ANSI SQL DEFAULTS ### @classmethod - def get_create_schema_sql(cls, project, schema): - if project.cfg.get('quoting', {}).get('schema', True): + def get_create_schema_sql(cls, project_cfg, schema): + if project_cfg.get('quoting', {}).get('schema', True): schema = cls.quote(schema) return ('create schema if not exists {schema}' .format(schema=schema)) @classmethod - def get_drop_schema_sql(cls, project, schema): - if project.cfg.get('quoting', {}).get('schema', True): + def get_drop_schema_sql(cls, project_cfg, schema): + if project_cfg.get('quoting', {}).get('schema', True): schema = cls.quote(schema) return ('drop schema if exists {schema} cascade' @@ -371,7 +371,7 @@ def get_drop_schema_sql(cls, project, schema): # although some adapters may override them ### @classmethod - def get_default_schema(cls, profile, project): + def get_default_schema(cls, profile, project_cfg): return profile.get('schema') @classmethod @@ -688,9 +688,9 @@ def execute_all(cls, profile, sqls, model_name=None): return connection @classmethod - def create_schema(cls, profile, project, schema, model_name=None): + def create_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Creating schema "%s".', schema) - sql = cls.get_create_schema_sql(project, schema) + sql = cls.get_create_schema_sql(project_cfg, schema) res = cls.add_query(profile, sql, model_name) cls.commit_if_has_connection(profile, model_name) @@ -698,14 +698,15 @@ def create_schema(cls, profile, project, schema, model_name=None): return res @classmethod - def drop_schema(cls, profile, project, schema, model_name=None): + def drop_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) - sql = cls.get_drop_schema_sql(project, schema) + sql = cls.get_drop_schema_sql(project_cfg, schema) return cls.add_query(profile, sql, model_name) @classmethod - def already_exists(cls, profile, project, schema, table, model_name=None): - relation = cls.get_relation(profile, schema=schema, identifier=table) + def already_exists(cls, profile, project_cfg, schema, table, model_name=None): + relation = cls.get_relation( + profile, project_cfg, schema=schema, identifier=table) return relation is not None @classmethod @@ -713,7 +714,7 @@ def quote(cls, identifier): return '"{}"'.format(identifier.replace('"', '""')) @classmethod - def quote_schema_and_table(cls, profile, project, + def quote_schema_and_table(cls, profile, project_cfg, schema, table, model_name=None): return '{}.{}'.format(cls.quote(schema), cls.quote(table)) diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index 64258e2dde4..17797dc9fe3 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -109,19 +109,20 @@ def alter_column_type(cls, profile, project, schema, table, column_name, 4. Rename the new column to existing column """ + relation = cls.Relation.create(schema=schema, identifier=table) + opts = { - "schema": schema, - "table": table, + "relation": relation, "old_column": column_name, "tmp_column": "{}__dbt_alter".format(column_name), "dtype": new_column_type } sql = """ - alter table {schema}.{table} add column "{tmp_column}" {dtype}; - update {schema}.{table} set "{tmp_column}" = "{old_column}"; - alter table {schema}.{table} drop column "{old_column}" cascade; - alter table {schema}.{table} rename column "{tmp_column}" to "{old_column}"; + alter table {relation} add column "{tmp_column}" {dtype}; + update {relation} set "{tmp_column}" = "{old_column}"; + alter table {relation} drop column "{old_column}" cascade; + alter table {relation} rename column "{tmp_column}" to "{old_column}"; """.format(**opts).strip() # noqa connection, cursor = cls.add_query(profile, sql, model_name) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 206ad2e3887..6774008620b 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -106,7 +106,7 @@ def open_connection(cls, connection): return result @classmethod - def list_relations(cls, profile, project, schema, model_name=None): + def list_relations(cls, profile, project_cfg, schema, model_name=None): sql = """ select table_name as name, table_schema as schema, table_type as type @@ -136,7 +136,7 @@ def list_relations(cls, profile, project, schema, model_name=None): for (name, _schema, type) in results] @classmethod - def rename_relation(cls, profile, project, from_relation, + def rename_relation(cls, profile, project_cfg, from_relation, to_relation, model_name=None): sql = 'alter table {} rename to {}'.format( from_relation, to_relation) @@ -148,7 +148,7 @@ def add_begin_query(cls, profile, name): return cls.add_query(profile, 'BEGIN', name, auto_begin=False) @classmethod - def get_existing_schemas(cls, profile, project, model_name=None): + def get_existing_schemas(cls, profile, project_cfg, model_name=None): sql = "select distinct schema_name from information_schema.schemata" connection, cursor = cls.add_query(profile, sql, model_name, @@ -158,7 +158,7 @@ def get_existing_schemas(cls, profile, project, model_name=None): return [row[0] for row in results] @classmethod - def check_schema_exists(cls, profile, project, schema, model_name=None): + def check_schema_exists(cls, profile, project_cfg, schema, model_name=None): sql = """ select count(*) from information_schema.schemata @@ -219,13 +219,13 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True, return connection, cursor @classmethod - def _make_match_kwargs(cls, project, schema, identifier): + def _make_match_kwargs(cls, project_cfg, schema, identifier): if identifier is not None and \ - project.cfg.get('quoting', {}).get('identifier', True) is False: + project_cfg.get('quoting', {}).get('identifier', True) is False: identifier = identifier.upper() if schema is not None and \ - project.cfg.get('quoting', {}).get('schema', True) is False: + project_cfg.get('quoting', {}).get('schema', True) is False: schema = schema.upper() return filter_null_values({'identifier': identifier, diff --git a/dbt/compilation.py b/dbt/compilation.py index 1a6f385b954..ab89bd54343 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -166,7 +166,7 @@ def compile_node(self, node, flat_graph): }) context = dbt.context.runtime.generate( - compiled_node, self.project.cfg, flat_graph) + compiled_node, self.project, flat_graph) compiled_node['compiled_sql'] = dbt.clients.jinja.get_rendered( node.get('raw_sql'), diff --git a/dbt/context/common.py b/dbt/context/common.py index fe56666fbc1..b472087d93b 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -279,13 +279,12 @@ def _return(value): raise dbt.exceptions.MacroReturn(value) -def get_this_relation(db_wrapper, project, profile, model): +def get_this_relation(db_wrapper, project_cfg, profile, model): table_name = dbt.utils.model_immediate_name( model, dbt.flags.NON_DESTRUCTIVE) return db_wrapper.adapter.Relation.create_from_node( - profile, model, table_name=table_name, - quote_policy=project.get('quoting', {})) + profile, model, table_name=table_name) def create_relation(relation_type, quoting_config): @@ -293,14 +292,30 @@ def create_relation(relation_type, quoting_config): class RelationWithContext(relation_type): @classmethod def create(cls, *args, **kwargs): + quote_policy = quoting_config + + if 'quote_policy' in kwargs: + quote_policy = dbt.utils.merge( + quote_policy, + kwargs.pop('quote_policy')) + return relation_type.create(*args, - quote_policy=quoting_config, + quote_policy=quote_policy, **kwargs) return RelationWithContext -def generate(model, project, flat_graph, provider=None): +def create_adapter(adapter_type, relation_type): + + class AdapterWithContext(adapter_type): + + Relation = relation_type + + return AdapterWithContext + + +def generate(model, project_cfg, flat_graph, provider=None): """ Not meant to be called directly. Call with either: dbt.context.parser.generate @@ -311,8 +326,8 @@ def generate(model, project, flat_graph, provider=None): raise dbt.exceptions.InternalException( "Invalid provider given to context: {}".format(provider)) - target_name = project.get('target') - profile = project.get('outputs').get(target_name) + target_name = project_cfg.get('target') + profile = project_cfg.get('outputs').get(target_name) target = profile.copy() target.pop('pass', None) target['name'] = target_name @@ -324,13 +339,18 @@ def generate(model, project, flat_graph, provider=None): pre_hooks = model.get('config', {}).get('pre-hook') post_hooks = model.get('config', {}).get('post-hook') - db_wrapper = DatabaseWrapper(model, adapter, profile, project) + relation_type = create_relation(adapter.Relation, + project_cfg.get('quoting')) + + db_wrapper = DatabaseWrapper(model, + create_adapter(adapter, relation_type), + profile, + project_cfg) context = dbt.utils.merge(context, { "adapter": db_wrapper, "api": { - "Relation": create_relation(adapter.Relation, - project.get('quoting')), + "Relation": relation_type, "Column": adapter.Column, }, "column": adapter.Column, @@ -348,7 +368,7 @@ def generate(model, project, flat_graph, provider=None): }, "post_hooks": post_hooks, "pre_hooks": pre_hooks, - "ref": provider.ref(model, project, profile, flat_graph), + "ref": provider.ref(db_wrapper, model, project_cfg, profile, flat_graph), "return": _return, "schema": model.get('schema', schema), "sql": model.get('injected_sql'), @@ -356,7 +376,7 @@ def generate(model, project, flat_graph, provider=None): "fromjson": fromjson, "tojson": tojson, "target": target, - "this": get_this_relation(db_wrapper, project, profile, model), + "this": get_this_relation(db_wrapper, project_cfg, profile, model), "try_or_compiler_error": try_or_compiler_error(model) }) @@ -368,7 +388,7 @@ def generate(model, project, flat_graph, provider=None): context = _add_macros(context, model, flat_graph) - context["write"] = write(model, project.get('target-path'), 'run') + context["write"] = write(model, project_cfg.get('target-path'), 'run') context["render"] = render(context, model) context["var"] = Var(model, context=context) context['context'] = context diff --git a/dbt/context/parser.py b/dbt/context/parser.py index 66bced09006..f43f55b6748 100644 --- a/dbt/context/parser.py +++ b/dbt/context/parser.py @@ -2,13 +2,11 @@ import dbt.context.common -from dbt.adapters.factory import get_adapter - execute = False -def ref(model, project, profile, flat_graph): +def ref(db_wrapper, model, project_cfg, profile, flat_graph): def ref(*args): if len(args) == 1 or len(args) == 2: @@ -17,10 +15,7 @@ def ref(*args): else: dbt.exceptions.ref_invalid_args(model, args) - adapter = get_adapter(profile) - return adapter.Relation.create_from_node( - profile, model, - quote_policy=project.get('quoting', {})) + return db_wrapper.adapter.Relation.create_from_node(profile, model) return ref @@ -52,6 +47,6 @@ def get(self, name, validator=None, default=None): return '' -def generate(model, project, flat_graph): +def generate(model, project_cfg, flat_graph): return dbt.context.common.generate( - model, project, flat_graph, dbt.context.parser) + model, project_cfg, flat_graph, dbt.context.parser) diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index f2248267659..efe0c375f1f 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -1,5 +1,3 @@ -from dbt.adapters.factory import get_adapter -from dbt.node_types import NodeType from dbt.utils import get_materialization, add_ephemeral_model_prefix import dbt.clients.jinja @@ -12,8 +10,9 @@ execute = True -def ref(model, project, profile, flat_graph): - current_project = project.get('name') +def ref(db_wrapper, model, project_cfg, profile, flat_graph): + current_project = project_cfg.get('name') + adapter = db_wrapper.adapter def do_ref(*args): target_model_name = None @@ -48,8 +47,6 @@ def do_ref(*args): is_ephemeral = (get_materialization(target_model) == 'ephemeral') - adapter = get_adapter(profile) - if is_ephemeral: model['extra_ctes'][target_model_id] = None return adapter.Relation.create( @@ -57,9 +54,7 @@ def do_ref(*args): identifier=add_ephemeral_model_prefix( target_model_name)).quote(identifier=False) else: - return adapter.Relation.create_from_node( - profile, target_model, - quote_policy=project.get('quoting', {})) + return adapter.Relation.create_from_node(profile, target_model) return do_ref @@ -97,6 +92,6 @@ def get(self, name, validator=None, default=None): return to_return -def generate(model, project, flat_graph): +def generate(model, project_cfg, flat_graph): return dbt.context.common.generate( - model, project, flat_graph, dbt.context.runtime) + model, project_cfg, flat_graph, dbt.context.runtime) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 807dbd29eae..fa2fdb2d7fa 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -320,7 +320,7 @@ def get_relation_returned_multiple_results(kwargs, matches): def approximate_relation_match(target, relation): raise_compiler_error( 'When searching for a relation, dbt found an approximate match. ' - 'Instead of guessing \nwhich relation to use, dbt will exit. ' + 'Instead of guessing \nwhich relation to use, dbt will move on. ' 'Please delete {relation}, or rename it to be less ambiguous.' '\nSearched for: {target}\nFound: {relation}' .format(target=target, diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/dbt/include/global_project/macros/materializations/archive/archive.sql index 738c2465130..8ac99e9ae7c 100644 --- a/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -1,30 +1,30 @@ -{% macro archive_select(source_schema, source_table, target_schema, target_table, unique_key, updated_at) %} +{% macro archive_select(source_relation, target_relation, unique_key, updated_at) %} with current_data as ( select - {% for col in adapter.get_columns_in_table(source_schema, source_table) %} + {% for col in adapter.get_columns_in_table(source_relation.schema, source_relation.identifier) %} "{{ col.name }}" {% if not loop.last %},{% endif %} {% endfor %}, {{ updated_at }} as "dbt_updated_at", {{ unique_key }} as "dbt_pk", {{ updated_at }} as "valid_from", null::timestamp as "tmp_valid_to" - from {{ source_schema }}.{{ source_table }} + from {{ source_relation }} ), archived_data as ( select - {% for col in adapter.get_columns_in_table(source_schema, source_table) %} + {% for col in adapter.get_columns_in_table(source_relation.schema, source_relation.identifier) %} "{{ col.name }}" {% if not loop.last %},{% endif %} {% endfor %}, {{ updated_at }} as "dbt_updated_at", {{ unique_key }} as "dbt_pk", "valid_from", "valid_to" as "tmp_valid_to" - from {{ target_schema }}.{{ target_table }} + from {{ target_relation }} ), @@ -76,17 +76,17 @@ {%- set target_schema = config.get('target_schema') -%} {%- set target_table = config.get('target_table') -%} + {%- set source_schema = config.get('source_schema') -%} + {%- set source_table = config.get('source_table') -%} + {%- set source_relation = adapter.get_relation( - schema=config.get('source_schema'), - identifier=config.get('source_table')) -%} + schema=source_schema, + identifier=source_table) -%} {%- set target_relation = adapter.get_relation( schema=target_schema, identifier=target_table) -%} - {%- set source_schema = config.get('source_schema') -%} - {%- set source_table = config.get('source_table') -%} - {%- if source_relation is none -%} {{ exceptions.missing_relation(source_relation) }} {%- endif -%} @@ -135,7 +135,7 @@ {% set tmp_table_sql -%} with dbt_archive_sbq as ( - {{ archive_select(source_schema, source_table, target_schema, target_table, unique_key, updated_at) }} + {{ archive_select(source_relation, target_relation, unique_key, updated_at) }} ) select * from dbt_archive_sbq @@ -150,15 +150,15 @@ to_table=target_table) }} {% call statement('main') -%} - update {{ target_schema }}.{{ identifier }} set "valid_to" = tmp."valid_to" - from {{ tmp_identifier }} as tmp - where tmp."scd_id" = {{ target_schema }}.{{ identifier }}."scd_id" + update {{ target_relation }} set "valid_to" = tmp."valid_to" + from {{ tmp_relation }} as tmp + where tmp."scd_id" = {{ target_relation }}."scd_id" and "change_type" = 'update'; - insert into {{ target_schema }}.{{ identifier }} ( + insert into {{ target_relation }} ( {{ column_list(dest_columns) }} ) - select {{ column_list(dest_columns) }} from {{ tmp_identifier }} + select {{ column_list(dest_columns) }} from {{ tmp_relation }} where "change_type" = 'insert'; {% endcall %} diff --git a/dbt/node_runners.py b/dbt/node_runners.py index a26ab388034..7da8080c6ce 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -409,7 +409,8 @@ def after_execute(self, result): self.print_result_line(result) def execute(self, model, flat_graph): - context = dbt.context.runtime.generate(model, self.project, flat_graph) + context = dbt.context.runtime.generate( + model, self.project.cfg, flat_graph) materialization_macro = dbt.utils.get_materialization_macro( flat_graph, diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index e48cd667969..8733126d786 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -83,9 +83,9 @@ def test__snowflake__simple_copy__quoting_on(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual('"seed"', '"view_model"') - self.assertTablesEqual('"seed"', '"incremental"') - self.assertTablesEqual('"seed"', '"materialized"') + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") self.use_default_project({ "data-paths": [self.dir("seed-update")], @@ -94,9 +94,9 @@ def test__snowflake__simple_copy__quoting_on(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual('"seed"', '"view_model"') - self.assertTablesEqual('"seed"', '"incremental"') - self.assertTablesEqual('"seed"', '"materialized"') + self.assertTablesEqual("seed", "view_model") + self.assertTablesEqual("seed", "incremental") + self.assertTablesEqual("seed", "materialized") @attr(type="snowflake") def test__snowflake__simple_copy__quoting_off(self): @@ -109,9 +109,9 @@ def test__snowflake__simple_copy__quoting_off(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed", "view_model") - self.assertTablesEqual("seed", "incremental") - self.assertTablesEqual("seed", "materialized") + self.assertTablesEqual("SEED", "VIEW_MODEL") + self.assertTablesEqual("SEED", "INCREMENTAL") + self.assertTablesEqual("SEED", "MATERIALIZED") self.use_default_project({ "data-paths": [self.dir("seed-update")], @@ -120,9 +120,9 @@ def test__snowflake__simple_copy__quoting_off(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertTablesEqual("seed", "view_model") - self.assertTablesEqual("seed", "incremental") - self.assertTablesEqual("seed", "materialized") + self.assertTablesEqual("SEED", "VIEW_MODEL") + self.assertTablesEqual("SEED", "INCREMENTAL") + self.assertTablesEqual("SEED", "MATERIALIZED") @attr(type="snowflake") def test__snowflake__seed__quoting_switch(self): diff --git a/test/integration/002_varchar_widening_test/test_varchar_widening.py b/test/integration/002_varchar_widening_test/test_varchar_widening.py index 34ddb89624a..e53a5d5ea0f 100644 --- a/test/integration/002_varchar_widening_test/test_varchar_widening.py +++ b/test/integration/002_varchar_widening_test/test_varchar_widening.py @@ -40,12 +40,12 @@ def test__snowflake__varchar_widening(self): self.run_dbt() - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("SEED", "incremental") + self.assertTablesEqual("SEED", "materialized") self.run_sql_file("test/integration/002_varchar_widening_test/update.sql") self.run_dbt() - self.assertTablesEqual("seed","incremental") - self.assertTablesEqual("seed","materialized") + self.assertTablesEqual("SEED", "incremental") + self.assertTablesEqual("SEED", "materialized") diff --git a/test/integration/003_simple_reference_test/test_simple_reference.py b/test/integration/003_simple_reference_test/test_simple_reference.py index b4efdcc58bc..2f0ab0a6916 100644 --- a/test/integration/003_simple_reference_test/test_simple_reference.py +++ b/test/integration/003_simple_reference_test/test_simple_reference.py @@ -18,7 +18,8 @@ def models(self): def test__postgres__simple_reference(self): self.use_profile('postgres') self.use_default_project() - self.run_sql_file("test/integration/003_simple_reference_test/seed.sql") + self.run_sql_file( + "test/integration/003_simple_reference_test/seed.sql") self.run_dbt() @@ -57,30 +58,31 @@ def test__snowflake__simple_reference(self): self.run_dbt() # Copies should match - self.assertTablesEqual("seed","incremental_copy") - self.assertTablesEqual("seed","materialized_copy") - self.assertTablesEqual("seed","view_copy") + self.assertTablesEqual("SEED", "incremental_copy") + self.assertTablesEqual("SEED", "materialized_copy") + self.assertTablesEqual("SEED", "view_copy") # Summaries should match - self.assertTablesEqual("summary_expected","incremental_summary") - self.assertTablesEqual("summary_expected","materialized_summary") - self.assertTablesEqual("summary_expected","view_summary") - self.assertTablesEqual("summary_expected","ephemeral_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "incremental_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "materialized_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "view_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "ephemeral_summary") - self.run_sql_file("test/integration/003_simple_reference_test/update.sql") + self.run_sql_file( + "test/integration/003_simple_reference_test/update.sql") self.run_dbt() # Copies should match - self.assertTablesEqual("seed","incremental_copy") - self.assertTablesEqual("seed","materialized_copy") - self.assertTablesEqual("seed","view_copy") + self.assertTablesEqual("SEED", "incremental_copy") + self.assertTablesEqual("SEED", "materialized_copy") + self.assertTablesEqual("SEED", "view_copy") # Summaries should match - self.assertTablesEqual("summary_expected","incremental_summary") - self.assertTablesEqual("summary_expected","materialized_summary") - self.assertTablesEqual("summary_expected","view_summary") - self.assertTablesEqual("summary_expected","ephemeral_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "incremental_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "materialized_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "view_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "ephemeral_summary") @attr(type='postgres') def test__postgres__simple_reference_with_models(self): @@ -145,10 +147,10 @@ def test__snowflake__simple_reference_with_models(self): self.run_dbt(['run', '--models', 'materialized_copy', 'ephemeral_copy']) # Copies should match - self.assertTablesEqual("seed","materialized_copy") + self.assertTablesEqual("SEED", "materialized_copy") created_models = self.get_models_in_schema() - self.assertTrue('MATERIALIZED_COPY' in created_models) + self.assertTrue('materialized_copy' in created_models) @attr(type='snowflake') def test__snowflake__simple_reference_with_models_and_children(self): @@ -162,11 +164,11 @@ def test__snowflake__simple_reference_with_models_and_children(self): self.run_dbt(['run', '--models', 'materialized_copy+', 'ephemeral_copy+']) # Copies should match - self.assertTablesEqual("seed","materialized_copy") + self.assertTablesEqual("SEED", "materialized_copy") # Summaries should match - self.assertTablesEqual("summary_expected","materialized_summary") - self.assertTablesEqual("summary_expected","ephemeral_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "materialized_summary") + self.assertTablesEqual("SUMMARY_EXPECTED", "ephemeral_summary") created_models = self.get_models_in_schema() @@ -178,10 +180,10 @@ def test__snowflake__simple_reference_with_models_and_children(self): # make sure this wasn't errantly materialized self.assertFalse('ephemeral_copy' in created_models) - self.assertTrue('MATERIALIZED_COPY' in created_models) - self.assertTrue('MATERIALIZED_SUMMARY' in created_models) - self.assertEqual(created_models['MATERIALIZED_COPY'], 'table') - self.assertEqual(created_models['MATERIALIZED_SUMMARY'], 'table') + self.assertTrue('materialized_copy' in created_models) + self.assertTrue('materialized_summary' in created_models) + self.assertEqual(created_models['materialized_copy'], 'table') + self.assertEqual(created_models['materialized_summary'], 'table') - self.assertTrue('EPHEMERAL_SUMMARY' in created_models) - self.assertEqual(created_models['EPHEMERAL_SUMMARY'], 'table') + self.assertTrue('ephemeral_summary' in created_models) + self.assertEqual(created_models['ephemeral_summary'], 'table') diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py index 308d85af5bc..c2e7a3c3d23 100644 --- a/test/integration/004_simple_archive_test/test_simple_archive.py +++ b/test/integration/004_simple_archive_test/test_simple_archive.py @@ -16,6 +16,11 @@ def models(self): @property def project_config(self): + source_table = 'seed' + + if self.adapter_type == 'snowflake': + source_table = source_table.upper() + return { "archive": [ { @@ -23,7 +28,7 @@ def project_config(self): "target_schema": self.unique_schema(), "tables": [ { - "source_table": "seed", + "source_table": source_table, "target_table": "archive_actual", "updated_at": '"updated_at"', "unique_key": '''"id" || '-' || "first_name"''' @@ -58,11 +63,11 @@ def test__snowflake__simple_archive(self): self.run_dbt(["archive"]) - self.assertTablesEqual("ARCHIVE_EXPECTED","ARCHIVE_ACTUAL") + self.assertTablesEqual("ARCHIVE_EXPECTED", "archive_actual") self.run_sql_file("test/integration/004_simple_archive_test/invalidate_snowflake.sql") self.run_sql_file("test/integration/004_simple_archive_test/update.sql") self.run_dbt(["archive"]) - self.assertTablesEqual("ARCHIVE_EXPECTED","ARCHIVE_ACTUAL") + self.assertTablesEqual("ARCHIVE_EXPECTED", "archive_actual") diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index 02444031b10..6d15d462b56 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -33,7 +33,7 @@ def test__snowflake__specific_model(self): self.run_dbt(['run', '--models', 'users']) - self.assertTablesEqual("seed", "users") + self.assertTablesEqual("SEED", "users") created_models = self.get_models_in_schema() self.assertFalse('users_rollup' in created_models) self.assertFalse('base_users' in created_models) @@ -62,8 +62,8 @@ def test__snowflake__specific_model_and_children(self): self.run_dbt(['run', '--models', 'users+']) - self.assertTablesEqual("seed", "users") - self.assertTablesEqual("summary_expected", "users_rollup") + self.assertTablesEqual("SEED", "users") + self.assertTablesEqual("SUMMARY_EXPECTED", "users_rollup") created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('emails' in created_models) @@ -91,8 +91,8 @@ def test__snowflake__specific_model_and_parents(self): self.run_dbt(['run', '--models', '+users_rollup']) - self.assertTablesEqual("seed", "users") - self.assertTablesEqual("summary_expected", "users_rollup") + self.assertTablesEqual("SEED", "users") + self.assertTablesEqual("SUMMARY_EXPECTED", "users_rollup") created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('emails' in created_models) @@ -120,7 +120,7 @@ def test__snowflake__specific_model_with_exclusion(self): self.run_dbt(['run', '--models', '+users_rollup', '--exclude', 'users_rollup']) - self.assertTablesEqual("seed", "users") + self.assertTablesEqual("SEED", "users") created_models = self.get_models_in_schema() self.assertFalse('base_users' in created_models) self.assertFalse('users_rollup' in created_models) diff --git a/test/integration/020_ephemeral_test/test_ephemeral.py b/test/integration/020_ephemeral_test/test_ephemeral.py index 682ad1eb767..4b56543bb45 100644 --- a/test/integration/020_ephemeral_test/test_ephemeral.py +++ b/test/integration/020_ephemeral_test/test_ephemeral.py @@ -1,6 +1,7 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest + class TestEphemeral(DBTIntegrationTest): def setUp(self): @@ -20,7 +21,7 @@ def test__postgres(self): self.use_default_project() self.run_sql_file("test/integration/020_ephemeral_test/seed.sql") - result = self.run_dbt() + self.run_dbt() self.assertTablesEqual("seed", "dependent") self.assertTablesEqual("seed", "double_dependent") @@ -33,5 +34,5 @@ def test__snowflake(self): self.run_dbt() - self.assertTablesEqual("seed", "dependent") - self.assertTablesEqual("seed", "double_dependent") + self.assertTablesEqual("SEED", "dependent") + self.assertTablesEqual("SEED", "double_dependent") diff --git a/test/integration/021_concurrency_test/test_concurrency.py b/test/integration/021_concurrency_test/test_concurrency.py index be42d46b34b..6a5449178d1 100644 --- a/test/integration/021_concurrency_test/test_concurrency.py +++ b/test/integration/021_concurrency_test/test_concurrency.py @@ -49,16 +49,16 @@ def test__snowflake__concurrency(self): self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view_model") - self.assertTablesEqual("seed", "dep") - self.assertTablesEqual("seed", "table_a") - self.assertTablesEqual("seed", "table_b") + self.assertTablesEqual("SEED", "view_model") + self.assertTablesEqual("SEED", "dep") + self.assertTablesEqual("SEED", "table_a") + self.assertTablesEqual("SEED", "table_b") self.run_sql_file("test/integration/021_concurrency_test/update.sql") self.run_dbt(expect_pass=False) - self.assertTablesEqual("seed", "view_model") - self.assertTablesEqual("seed", "dep") - self.assertTablesEqual("seed", "table_a") - self.assertTablesEqual("seed", "table_b") + self.assertTablesEqual("SEED", "view_model") + self.assertTablesEqual("SEED", "dep") + self.assertTablesEqual("SEED", "table_a") + self.assertTablesEqual("SEED", "table_b") diff --git a/test/integration/base.py b/test/integration/base.py index 02778ea6c06..32982d56f46 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -121,8 +121,14 @@ def bigquery_profile(self): } def unique_schema(self): - schema = self.schema - return "{}_{}".format(self.prefix, schema) + schema = self.schema + + to_return = "{}_{}".format(self.prefix, schema) + + if self.adapter_type == 'snowflake': + return to_return.upper() + + return to_return.lower() def get_profile(self, adapter_type): if adapter_type == 'postgres': @@ -133,6 +139,8 @@ def get_profile(self, adapter_type): return self.bigquery_profile() def setUp(self): + self.adapter_type = 'postgres' + # create a dbt_project.yml base_project_config = { @@ -190,8 +198,9 @@ def setUp(self): adapter.drop_schema(profile, project, schema_name, '__test') adapter.create_schema(profile, project, schema_name, '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) - self.run_sql('CREATE SCHEMA {}'.format(self.unique_schema())) + schema = self.adapter.quote(self.unique_schema()) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(schema)) + self.run_sql('CREATE SCHEMA {}'.format(schema)) def use_default_project(self, overrides=None): # create a dbt_project.yml @@ -215,6 +224,8 @@ def use_default_project(self, overrides=None): yaml.safe_dump(project_config, f, default_flow_style=True) def use_profile(self, adapter_type): + self.adapter_type = adapter_type + profile_config = {} default_profile_config = self.get_profile(adapter_type) @@ -246,8 +257,9 @@ def use_profile(self, adapter_type): adapter.create_schema(profile, self.project, self.unique_schema(), '__test') else: - self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(self.unique_schema())) - self.run_sql('CREATE SCHEMA {}'.format(self.unique_schema())) + schema = self.adapter.quote(self.unique_schema()) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(schema)) + self.run_sql('CREATE SCHEMA {}'.format(schema)) def tearDown(self): os.remove(DBT_PROFILES) @@ -266,11 +278,8 @@ def tearDown(self): adapter.drop_schema(self._profile, self.project, self.unique_schema(), '__test') else: - if self.project.cfg.get('quoting', {}).get('schema', True): - schema = self.adapter.quote(self.unique_schema()) - - self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE' - .format(schema)) + schema = self.adapter.quote(self.unique_schema()) + self.run_sql('DROP SCHEMA IF EXISTS {} CASCADE'.format(schema)) self.handle.close() # hack for BQ -- TODO @@ -385,14 +394,6 @@ def _assertTablesEqualSql(self, table_a_schema, table_a, table_b_schema, table_b else: columns_csv = ", ".join(['{}'.format(record[0]) for record in columns]) - if self.project.cfg.get('quoting', {}).get('identifier', True): - table_a = self.adapter.quote(table_a) - table_b = self.adapter.quote(table_b) - - if self.project.cfg.get('quoting', {}).get('schema', True): - table_a_schema = self.adapter.quote(table_a_schema) - table_b_schema = self.adapter.quote(table_b_schema) - sql = """ SELECT COUNT(*) FROM ( (SELECT {columns} FROM {table_a_schema}.{table_a} EXCEPT @@ -402,10 +403,10 @@ def _assertTablesEqualSql(self, table_a_schema, table_a, table_b_schema, table_b SELECT {columns} FROM {table_a_schema}.{table_a}) ) AS a""".format( columns=columns_csv, - table_a_schema=table_a_schema, - table_b_schema=table_b_schema, - table_a=table_a, - table_b=table_b + table_a_schema=self.adapter.quote(table_a_schema), + table_b_schema=self.adapter.quote(table_b_schema), + table_a=self.adapter.quote(table_a), + table_b=self.adapter.quote(table_b) ) return sql @@ -441,20 +442,14 @@ def assertTableRowCountsEqual(self, table_a, table_b, table_b_schema = self.unique_schema() \ if table_b_schema is None else table_b_schema - if self.project.cfg.get('quoting', {}).get('identifier', True): - table_a = self.adapter.quote(table_a) - table_b = self.adapter.quote(table_b) - - if self.project.cfg.get('quoting', {}).get('schema', True): - table_a_schema = self.adapter.quote(table_a_schema) - table_b_schema = self.adapter.quote(table_b_schema) - table_a_result = self.run_sql( 'SELECT COUNT(*) FROM {}.{}' - .format(table_a_schema, table_a), fetch='one') + .format(self.adapter.quote(table_a_schema), + self.adapter.quote(table_a)), fetch='one') table_b_result = self.run_sql( 'SELECT COUNT(*) FROM {}.{}' - .format(table_b_schema, table_b), fetch='one') + .format(self.adapter.quote(table_b_schema), + self.adapter.quote(table_b)), fetch='one') self.assertEquals( table_a_result[0], @@ -487,14 +482,6 @@ def assertTableColumnsEqual(self, table_a, table_b, table_a_schema=None, table_b table_a_schema = self.unique_schema() if table_a_schema is None else table_a_schema table_b_schema = self.unique_schema() if table_b_schema is None else table_b_schema - if self.project.cfg.get('quoting', {}).get('identifier', True): - table_a = self.adapter.quote(table_a) - table_b = self.adapter.quote(table_b) - - if self.project.cfg.get('quoting', {}).get('schema', True): - table_a_schema = self.adapter.quote(table_a_schema) - table_b_schema = self.adapter.quote(table_b_schema) - table_a_result = self.get_table_columns(table_a, table_a_schema) table_b_result = self.get_table_columns(table_b, table_b_schema) From 126ab2307414853fab15270e9e611e4a46090607 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 25 Apr 2018 18:47:18 -0400 Subject: [PATCH 55/61] add already_exists tests --- .../models/advanced_incremental.sql | 16 ++++++++++++++++ .../models/incremental.sql | 6 ++++++ 2 files changed, 22 insertions(+) create mode 100644 test/integration/001_simple_copy_test/models/advanced_incremental.sql diff --git a/test/integration/001_simple_copy_test/models/advanced_incremental.sql b/test/integration/001_simple_copy_test/models/advanced_incremental.sql new file mode 100644 index 00000000000..0587040a58f --- /dev/null +++ b/test/integration/001_simple_copy_test/models/advanced_incremental.sql @@ -0,0 +1,16 @@ +{{ + config( + materialized = "incremental", + sql_where = "TRUE" + ) +}} + + +select * +from {{ ref('seed') }} + +{% if adapter.already_exists(this.schema, this.table) and not flags.FULL_REFRESH %} + + where id > (select max(id) from {{this}}) + +{% endif %} diff --git a/test/integration/002_varchar_widening_test/models/incremental.sql b/test/integration/002_varchar_widening_test/models/incremental.sql index d7e63d2a5d1..10141cd3f13 100644 --- a/test/integration/002_varchar_widening_test/models/incremental.sql +++ b/test/integration/002_varchar_widening_test/models/incremental.sql @@ -6,3 +6,9 @@ }} select * from {{ this.schema }}.seed + +{% if adapter.already_exists(this.schema, this.table) %} + + where id > (select max(id) from {{this}}) + +{% endif %} From e564dee591af5d4a34c5ddd908af7aa91ad9eb8e Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 25 Apr 2018 21:30:51 -0400 Subject: [PATCH 56/61] fix for incremental models, plus test --- .../macros/materializations/incremental/incremental.sql | 2 +- .../001_simple_copy_test/models/advanced_incremental.sql | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql index 0bf8c7bff30..71945f29f08 100644 --- a/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -7,7 +7,7 @@ from {{ target_relation }} where ({{ unique_key }}) in ( select ({{ unique_key }}) - from {{ tmp_relation }} + from {{ tmp_relation.include(schema=False) }} ); {%- endmacro %} diff --git a/test/integration/001_simple_copy_test/models/advanced_incremental.sql b/test/integration/001_simple_copy_test/models/advanced_incremental.sql index 0587040a58f..0072463c1a0 100644 --- a/test/integration/001_simple_copy_test/models/advanced_incremental.sql +++ b/test/integration/001_simple_copy_test/models/advanced_incremental.sql @@ -1,7 +1,8 @@ {{ config( materialized = "incremental", - sql_where = "TRUE" + sql_where = "TRUE", + unique_key = "id" ) }} From d33ec80251cbaa7e23d9d17e2f1c55e345734363 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 26 Apr 2018 09:54:47 -0400 Subject: [PATCH 57/61] fix unit tests, pep8 --- dbt/adapters/bigquery/impl.py | 39 ++++++++++++++++++++++------------ dbt/adapters/default/impl.py | 28 ++++++++++++++---------- dbt/adapters/snowflake/impl.py | 3 ++- dbt/context/common.py | 3 ++- dbt/project.py | 1 + test/unit/test_parser.py | 2 ++ 6 files changed, 49 insertions(+), 27 deletions(-) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index de62e412535..430a12eb16d 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -181,7 +181,8 @@ def list_relations(cls, profile, project_cfg, schema, model_name=None): credentials = connection.get('credentials', {}) client = connection.get('handle') - bigquery_dataset = cls.get_dataset(profile, project_cfg, schema, model_name) + bigquery_dataset = cls.get_dataset( + profile, project_cfg, schema, model_name) all_tables = client.list_tables(bigquery_dataset) relation_types = { @@ -206,12 +207,14 @@ def drop_relation(cls, profile, project_cfg, relation, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, project_cfg, relation.schema, model_name) + dataset = cls.get_dataset( + profile, project_cfg, relation.schema, model_name) relation_object = dataset.table(relation.identifier) client.delete_table(relation_object) @classmethod - def rename(cls, profile, project_cfg, schema, from_name, to_name, model_name=None): + def rename(cls, profile, project_cfg, schema, + from_name, to_name, model_name=None): raise dbt.exceptions.NotImplementedException( '`rename` is not implemented for this adapter!') @@ -267,7 +270,8 @@ def make_date_partitioned_table(cls, profile, project_cfg, dataset_name, conn = cls.get_connection(profile, model_name) client = conn.get('handle') - dataset = cls.get_dataset(profile, project_cfg, dataset_name, identifier) + dataset = cls.get_dataset(profile, project_cfg, + dataset_name, identifier) table_ref = dataset.table(identifier) table = google.cloud.bigquery.Table(table_ref) table.partitioning_type = 'DAY' @@ -275,8 +279,8 @@ def make_date_partitioned_table(cls, profile, project_cfg, dataset_name, return client.create_table(table) @classmethod - def materialize_as_table(cls, profile, project_cfg, dataset, model, model_sql, - decorator=None): + def materialize_as_table(cls, profile, project_cfg, dataset, + model, model_sql, decorator=None): model_name = model.get('name') conn = cls.get_connection(profile, model_name) @@ -302,7 +306,8 @@ def materialize_as_table(cls, profile, project_cfg, dataset, model, model_sql, return "CREATE TABLE" @classmethod - def execute_model(cls, profile, project_cfg, model, materialization, sql_override=None, + def execute_model(cls, profile, project_cfg, model, + materialization, sql_override=None, decorator=None, model_name=None): if sql_override is None: @@ -315,13 +320,15 @@ def execute_model(cls, profile, project_cfg, model, materialization, sql_overrid model_name = model.get('name') model_schema = model.get('schema') - dataset = cls.get_dataset(profile, project_cfg, model_schema, model_name) + dataset = cls.get_dataset(profile, project_cfg, + model_schema, model_name) if materialization == 'view': res = cls.materialize_as_view(profile, project_cfg, dataset, model) elif materialization == 'table': - res = cls.materialize_as_table(profile, project_cfg, dataset, model, - sql_override, decorator) + res = cls.materialize_as_table( + profile, project_cfg, dataset, model, + sql_override, decorator) else: msg = "Invalid relation type: '{}'".format(materialization) raise dbt.exceptions.RuntimeException(msg, model) @@ -391,7 +398,8 @@ def drop_tables_in_schema(cls, profile, project_cfg, dataset): def drop_schema(cls, profile, project_cfg, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) - if not cls.check_schema_exists(profile, project_cfg, schema, model_name): + if not cls.check_schema_exists(profile, project_cfg, + schema, model_name): return conn = cls.get_connection(profile) @@ -412,7 +420,8 @@ def get_existing_schemas(cls, profile, project_cfg, model_name=None): return [ds.dataset_id for ds in all_datasets] @classmethod - def get_columns_in_table(cls, profile, project_cfg, schema_name, table_name, + def get_columns_in_table(cls, profile, project_cfg, + schema_name, table_name, database=None, model_name=None): # BigQuery does not have databases -- the database parameter is here @@ -439,7 +448,8 @@ def get_columns_in_table(cls, profile, project_cfg, schema_name, table_name, return columns @classmethod - def check_schema_exists(cls, profile, project_cfg, schema, model_name=None): + def check_schema_exists(cls, profile, project_cfg, + schema, model_name=None): conn = cls.get_connection(profile, model_name) client = conn.get('handle') @@ -479,7 +489,8 @@ def quote(cls, identifier): return '`{}`'.format(identifier) @classmethod - def quote_schema_and_table(cls, profile, project_cfg, schema, table, model_name=None): + def quote_schema_and_table(cls, profile, project_cfg, schema, + table, model_name=None): return cls.render_relation(profile, project_cfg, cls.quote(schema), cls.quote(table)) diff --git a/dbt/adapters/default/impl.py b/dbt/adapters/default/impl.py index 9bad2c47152..cfe474434f2 100644 --- a/dbt/adapters/default/impl.py +++ b/dbt/adapters/default/impl.py @@ -91,13 +91,14 @@ def get_status(cls, cursor): '`get_status` is not implemented for this adapter!') @classmethod - def alter_column_type(cls, profile, project_cfg, schema, table, column_name, - new_column_type, model_name=None): + def alter_column_type(cls, profile, project_cfg, schema, table, + column_name, new_column_type, model_name=None): raise dbt.exceptions.NotImplementedException( '`alter_column_type` is not implemented for this adapter!') @classmethod - def query_for_existing(cls, profile, project_cfg, schemas, model_name=None): + def query_for_existing(cls, profile, project_cfg, schemas, + model_name=None): if not isinstance(schemas, (list, tuple)): schemas = [schemas] @@ -169,10 +170,12 @@ def truncate(cls, profile, project_cfg, schema, table, model_name=None): identifier=table, type='table') - return cls.truncate_relation(profile, project_cfg, relation, model_name) + return cls.truncate_relation(profile, project_cfg, + relation, model_name) @classmethod - def truncate_relation(cls, profile, project_cfg, relation, model_name=None): + def truncate_relation(cls, profile, project_cfg, + relation, model_name=None): sql = 'truncate table {}'.format(relation) connection, cursor = cls.add_query(profile, sql, model_name) @@ -247,8 +250,8 @@ def _get_columns_in_table_sql(cls, schema_name, table_name, database): return sql @classmethod - def get_columns_in_table(cls, profile, project_cfg, schema_name, table_name, - database=None, model_name=None): + def get_columns_in_table(cls, profile, project_cfg, schema_name, + table_name, database=None, model_name=None): sql = cls._get_columns_in_table_sql(schema_name, table_name, database) connection, cursor = cls.add_query( profile, sql, model_name) @@ -279,7 +282,8 @@ def expand_target_column_types(cls, profile, project_cfg, target_columns = cls._table_columns_to_dict( cls.get_columns_in_table( - profile, project_cfg, to_schema, to_table, model_name=model_name)) + profile, project_cfg, to_schema, to_table, + model_name=model_name)) for column_name, reference_column in reference_columns.items(): target_column = target_columns.get(column_name) @@ -294,8 +298,9 @@ def expand_target_column_types(cls, profile, project_cfg, to_schema, to_table) - cls.alter_column_type(profile, project_cfg, to_schema, to_table, - column_name, new_type, model_name) + cls.alter_column_type(profile, project_cfg, to_schema, + to_table, column_name, new_type, + model_name) ### # RELATIONS @@ -704,7 +709,8 @@ def drop_schema(cls, profile, project_cfg, schema, model_name=None): return cls.add_query(profile, sql, model_name) @classmethod - def already_exists(cls, profile, project_cfg, schema, table, model_name=None): + def already_exists(cls, profile, project_cfg, + schema, table, model_name=None): relation = cls.get_relation( profile, project_cfg, schema=schema, identifier=table) return relation is not None diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 6774008620b..72836cf2518 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -158,7 +158,8 @@ def get_existing_schemas(cls, profile, project_cfg, model_name=None): return [row[0] for row in results] @classmethod - def check_schema_exists(cls, profile, project_cfg, schema, model_name=None): + def check_schema_exists(cls, profile, project_cfg, + schema, model_name=None): sql = """ select count(*) from information_schema.schemata diff --git a/dbt/context/common.py b/dbt/context/common.py index b472087d93b..7342aa6dcc8 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -368,7 +368,8 @@ def generate(model, project_cfg, flat_graph, provider=None): }, "post_hooks": post_hooks, "pre_hooks": pre_hooks, - "ref": provider.ref(db_wrapper, model, project_cfg, profile, flat_graph), + "ref": provider.ref(db_wrapper, model, project_cfg, + profile, flat_graph), "return": _return, "schema": model.get('schema', schema), "sql": model.get('injected_sql'), diff --git a/dbt/project.py b/dbt/project.py index a16e38e49ef..ab9ecd14405 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -27,6 +27,7 @@ 'outputs': {'default': {}}, 'target': 'default', 'models': {}, + 'quoting': {}, 'profile': None, 'packages': [], 'modules-path': 'dbt_modules' diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 86b8fd2fe28..570c48c4ba0 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -28,6 +28,7 @@ def setUp(self): 'profile': 'test', 'project-root': os.path.abspath('.'), 'target': 'test', + 'quoting': {}, 'outputs': { 'test': { 'type': 'postgres', @@ -42,6 +43,7 @@ def setUp(self): 'version': '0.1', 'project-root': os.path.abspath('./dbt_modules/snowplow'), 'target': 'test', + 'quoting': {}, 'outputs': { 'test': { 'type': 'postgres', From 124e14bf3b23981a856562446ccd126556792907 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 26 Apr 2018 13:01:56 -0400 Subject: [PATCH 58/61] self code review --- TODO.md | 20 -------------------- dbt/adapters/bigquery/impl.py | 1 - dbt/adapters/default/relation.py | 6 +++--- dbt/adapters/snowflake/relation.py | 2 -- dbt/deprecations.py | 14 +------------- 5 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 TODO.md diff --git a/TODO.md b/TODO.md deleted file mode 100644 index d6545ae1abf..00000000000 --- a/TODO.md +++ /dev/null @@ -1,20 +0,0 @@ -- [ ] rip out query_for_existing > replace with get_relation - - maybe part of the relation object is metadata about when the relation was pulled - from the db (relation type, when we asked the db about it, columns, ??) -- [ ] add get_relation, list_relations -- fns / macros - - [x] query_for_existing - - [ ] get_columns_in_table - - [x] rename - - [x] truncate - - [x] drop - - [ ] reset_csv_table - - [ ] load_csv_rows - - [ ] handle_csv_table - - [ ] make_date_partitioned_table - - [x] macro: get_existing_relation_type - - [x] macro: create_table_as - - [x] macro: create_view_as - - [x] macro: create_archive_table - - (go back through these after BQ pr is merged) -- [ ] deprecation warnings diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 430a12eb16d..e4d0dd39239 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -31,7 +31,6 @@ class BigQueryAdapter(PostgresAdapter): "query_for_existing", "execute_model", "drop", - "drop_relation", "execute", "quote_schema_and_table", "make_date_partitioned_table", diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index a3a5c561050..a15f11be169 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -85,7 +85,8 @@ def matches(self, database=None, schema=None, identifier=None): if not search: # nothing was passed in - pass + raise dbt.exceptions.RuntimeException( + "Tried to match relation, but no search path was passed!") exact_match = True approximate_match = True @@ -152,8 +153,7 @@ def render(self, use_table_name=True): self.should_quote(k))) if len(parts) == 0: - # TODO - raise RuntimeError( + raise dbt.exceptions.RuntimeException( "No path parts are included! Nothing to render.") return '.'.join(parts) diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index 65b0c83b6e3..a94a6803486 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -48,5 +48,3 @@ def create_from_node(cls, profile, node, **kwargs): identifier=node.get('name'), **kwargs) - def get_path_part(self, part): - return self.path.get(part) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index d497e7c3f34..00d656b7be0 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -33,17 +33,6 @@ class SeedDropExistingDeprecation(DBTDeprecation): will be removed in a future version of dbt.""" -class NonRelationAdapterFunctionDeprecation(DBTDeprecation): - # TODO add link to docs - name = 'relations-api' - description = """ - Your project uses a deprecated adapter function '{fn}'. - Please use the Relations API instead. You can find - more information in the documentation. - (ADD LINK) - """ - - def warn(name, *args, **kwargs): if name not in deprecations: # this should (hopefully) never happen @@ -61,8 +50,7 @@ def warn(name, *args, **kwargs): deprecations_list = [ DBTRepositoriesDeprecation(), - SeedDropExistingDeprecation(), - NonRelationAdapterFunctionDeprecation(), + SeedDropExistingDeprecation() ] deprecations = {d.name: d for d in deprecations_list} From 23d2a55899490cb9f56d4d30e50ae80161b280e2 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 26 Apr 2018 13:03:57 -0400 Subject: [PATCH 59/61] fix project var name from merge conflict --- dbt/context/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/context/common.py b/dbt/context/common.py index 73e1e68601c..32a6b000d52 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -353,7 +353,7 @@ def generate(model, project_cfg, flat_graph, provider=None): profile, project_cfg) - cli_var_overrides = project.get('cli_vars', {}) + cli_var_overrides = project_cfg.get('cli_vars', {}) context = dbt.utils.merge(context, { "adapter": db_wrapper, From 692cc5573568a882ae1440c8e15ce24c4eb609c1 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 26 Apr 2018 13:27:25 -0400 Subject: [PATCH 60/61] pep8 --- dbt/adapters/snowflake/relation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index a94a6803486..2f8063ceede 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -47,4 +47,3 @@ def create_from_node(cls, profile, node, **kwargs): schema=node.get('schema'), identifier=node.get('name'), **kwargs) - From c37ded2bd3c4a57a1654e7f524b38bec0b2a4b5c Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 26 Apr 2018 13:42:15 -0400 Subject: [PATCH 61/61] fix relation schemas --- dbt/adapters/bigquery/relation.py | 11 +++++++---- dbt/adapters/default/relation.py | 11 +++++++---- dbt/adapters/snowflake/relation.py | 11 +++++++---- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index bfd366d02e3..5c8f130c71e 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -6,7 +6,7 @@ class BigQueryRelation(DefaultRelation): DEFAULTS = { 'metadata': { - '_type': 'BigQueryRelation' + 'type': 'BigQueryRelation' }, 'quote_character': '`', 'quote_policy': { @@ -45,9 +45,12 @@ class BigQueryRelation(DefaultRelation): 'type': 'object', 'properties': { 'metadata': { - '_type': { - 'type': 'string', - 'const': 'BigQueryRelation', + 'type': 'object', + 'properties': { + 'type': { + 'type': 'string', + 'const': 'BigQueryRelation', + }, }, }, 'type': { diff --git a/dbt/adapters/default/relation.py b/dbt/adapters/default/relation.py index a15f11be169..869a022a856 100644 --- a/dbt/adapters/default/relation.py +++ b/dbt/adapters/default/relation.py @@ -18,7 +18,7 @@ class DefaultRelation(APIObject): DEFAULTS = { 'metadata': { - '_type': 'DefaultRelation' + 'type': 'DefaultRelation' }, 'quote_character': '"', 'quote_policy': { @@ -57,9 +57,12 @@ class DefaultRelation(APIObject): 'type': 'object', 'properties': { 'metadata': { - '_type': { - 'type': 'string', - 'const': 'DefaultRelation', + 'type': 'object', + 'properties': { + 'type': { + 'type': 'string', + 'const': 'DefaultRelation', + }, }, }, 'type': { diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index 2f8063ceede..d8765c54775 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -4,7 +4,7 @@ class SnowflakeRelation(DefaultRelation): DEFAULTS = { 'metadata': { - '_type': 'SnowflakeRelation' + 'type': 'SnowflakeRelation' }, 'quote_character': '"', 'quote_policy': { @@ -23,9 +23,12 @@ class SnowflakeRelation(DefaultRelation): 'type': 'object', 'properties': { 'metadata': { - '_type': { - 'type': 'string', - 'const': 'SnowflakeRelation', + 'type': 'object', + 'properties': { + 'type': { + 'type': 'string', + 'const': 'SnowflakeRelation', + }, }, }, 'type': {