From 1313445afebc04e808bfeab8d17096e9cc1fe7a3 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 14 Apr 2020 11:25:36 -0600 Subject: [PATCH] change list_relations_without_caching to use "show objects in schema" --- CHANGELOG.md | 3 +- .../snowflake/dbt/adapters/snowflake/impl.py | 45 ++++++++++++++++++- .../dbt/include/snowflake/macros/adapters.sql | 32 ++++++------- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d41d179294..f9aa744bc3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Added support for `db_groups` and `autocreate` flags in Redshift configurations. ([#1995](https://github.com/fishtown-analytics/dbt/issues/1995), [#2262](https://github.com/fishtown-analytics/dbt/pull/2262)) - Users can supply paths as arguments to `--models` and `--select`, either explicitily by prefixing with `path:` or implicitly with no prefix. ([#454](https://github.com/fishtown-analytics/dbt/issues/454), [#2258](https://github.com/fishtown-analytics/dbt/pull/2258)) - dbt now builds the relation cache for "dbt compile" and "dbt ls" as well as "dbt run" ([#1705](https://github.com/fishtown-analytics/dbt/issues/1705), [#2319](https://github.com/fishtown-analytics/dbt/pull/2319)) +- Snowflake now uses "show terse objects" to build the relations cache instead of selecting from the information schema ([#2174](https://github.com/fishtown-analytics/dbt/issues/2174), [#2322](https://github.com/fishtown-analytics/dbt/pull/2322)) ### Fixes - When a jinja value is undefined, give a helpful error instead of failing with cryptic "cannot pickle ParserMacroCapture" errors ([#2110](https://github.com/fishtown-analytics/dbt/issues/2110), [#2184](https://github.com/fishtown-analytics/dbt/pull/2184)) @@ -28,7 +29,7 @@ Contributors: - [@rodrigodelmonte](https://github.com/rodrigodelmonte) [#2298](https://github.com/fishtown-analytics/dbt/pull/2298) - [@sumanau7](https://github.com/sumanau7) ([#2279](https://github.com/fishtown-analytics/dbt/pull/2279), [#2263](https://github.com/fishtown-analytics/dbt/pull/2263), [#2297](https://github.com/fishtown-analytics/dbt/pull/2297)) -## dbt 0.16.1 (Relase date TBD) +## dbt 0.16.1 (April 14, 2020) ### Features - Support for appending query comments to SQL queries. ([#2138](https://github.com/fishtown-analytics/dbt/issues/2138) [#2199](https://github.com/fishtown-analytics/dbt/issues/2199)) diff --git a/plugins/snowflake/dbt/adapters/snowflake/impl.py b/plugins/snowflake/dbt/adapters/snowflake/impl.py index 37e27dcb2de..6c726eadc6c 100644 --- a/plugins/snowflake/dbt/adapters/snowflake/impl.py +++ b/plugins/snowflake/dbt/adapters/snowflake/impl.py @@ -3,7 +3,10 @@ import agate from dbt.adapters.sql import SQLAdapter -from dbt.adapters.sql.impl import LIST_SCHEMAS_MACRO_NAME +from dbt.adapters.sql.impl import ( + LIST_SCHEMAS_MACRO_NAME, + LIST_RELATIONS_MACRO_NAME, +) from dbt.adapters.snowflake import SnowflakeConnectionManager from dbt.adapters.snowflake import SnowflakeRelation from dbt.adapters.snowflake import SnowflakeColumn @@ -99,3 +102,43 @@ def list_schemas(self, database: str) -> List[str]: # want is 'name' return [row['name'] for row in results] + + def list_relations_without_caching( + self, information_schema, schema + ) -> List[SnowflakeRelation]: + kwargs = {'information_schema': information_schema, 'schema': schema} + try: + results = self.execute_macro( + LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs + ) + except DatabaseException as exc: + # if the schema doesn't exist, we just want to return. + # Alternatively, we could query the list of schemas before we start + # and skip listing the missing ones, which sounds expensive. + if 'Object does not exist' in str(exc): + return [] + raise + + relations = [] + quote_policy = { + 'database': True, + 'schema': True, + 'identifier': True + } + + columns = ['database_name', 'schema_name', 'name', 'kind'] + for _database, _schema, _identifier, _type in results.select(columns): + try: + _type = self.Relation.get_relation_type(_type.lower()) + except ValueError: + _type = self.Relation.External + relations.append(self.Relation.create( + database=_database, + schema=_schema, + identifier=_identifier, + quote_policy=quote_policy, + type=_type + )) + + return relations diff --git a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql index 9d051c4e39c..38296ec9510 100644 --- a/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql +++ b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql @@ -99,22 +99,22 @@ {% macro snowflake__list_relations_without_caching(information_schema, schema) %} - {% call statement('list_relations_without_caching', fetch_result=True) -%} - select - table_catalog as database, - table_name as name, - table_schema as schema, - case when table_type = 'BASE TABLE' then 'table' - when table_type = 'VIEW' then 'view' - when table_type = 'MATERIALIZED VIEW' then 'materializedview' - when table_type = 'EXTERNAL TABLE' then 'external' - else table_type - end as table_type - from {{ information_schema }}.tables - where table_schema ilike '{{ schema }}' - and table_catalog ilike '{{ information_schema.database.lower() }}' - {% endcall %} - {{ return(load_result('list_relations_without_caching').table) }} + {%- set db_name = adapter.quote_as_configured(information_schema.database, 'database') -%} + {%- set schema_name = adapter.quote_as_configured(schema, 'schema') -%} + {%- set sql -%} + show terse objects in {{ db_name }}.{{ schema_name }} + {%- endset -%} + + {%- set result = run_query(sql) -%} + {% set maximum = 10000 %} + {% if (result | length) >= maximum %} + {% set msg %} + Too many schemas in schema {{ database }}.{{ schema }}! dbt can only get + information about schemas with fewer than {{ maximum }} objects. + {% endset %} + {% do exceptions.raise_compiler_error(msg) %} + {% endif %} + {%- do return(result) -%} {% endmacro %}