Snowflake catalog/manifest support (#832)

dbt/adapters/default/impl.py

@@ -17,6 +17,9 @@
 
 from dbt.adapters.default.relation import DefaultRelation
 
+GET_CATALOG_OPERATION_NAME = 'get_catalog_data'
+GET_CATALOG_RESULT_KEY = 'catalog' # defined in get_catalog() macro
+
 lock = multiprocessing.Lock()
 connections_in_use = {}
 connections_available = []
@@ -809,5 +812,12 @@ def run_operation(cls, profile, project_cfg, manifest, operation_name,
  ###
  @classmethod
  def get_catalog(cls, profile, project_cfg, manifest):
- raise dbt.exceptions.NotImplementedException(
- '`get_catalog` is not implemented for this adapter!')
+ results = cls.run_operation(profile, project_cfg, manifest,
+ GET_CATALOG_OPERATION_NAME,
+ GET_CATALOG_RESULT_KEY)
+ schemas = list({
+ node.to_dict()['schema']
+ for node in manifest.nodes.values()
+ })
+ results = results.table.where(lambda r: r['table_schema'] in schemas)
+ return results

dbt/adapters/postgres/impl.py

@@ -9,9 +9,6 @@
 
 from dbt.logger import GLOBAL_LOGGER as logger
 
-GET_CATALOG_OPERATION_NAME = 'get_catalog_data'
-GET_CATALOG_RESULT_KEY = 'catalog' # defined in get_catalog() macro
-
 
 class PostgresAdapter(dbt.adapters.default.DefaultAdapter):
 
@@ -220,14 +217,3 @@ def convert_date_type(cls, agate_table, col_idx):
  @classmethod
  def convert_time_type(cls, agate_table, col_idx):
  return "time"
-
- @classmethod
- def get_catalog(cls, profile, project_cfg, manifest):
- results = cls.run_operation(profile, project_cfg, manifest,
- GET_CATALOG_OPERATION_NAME,
- GET_CATALOG_RESULT_KEY)
-
- schemas = cls.get_existing_schemas(profile, project_cfg)
- results = results.table.where(lambda r: r['table_schema'] in schemas)
-
- return results

dbt/include/global_project/macros/adapters/common.sql

@@ -119,8 +119,46 @@
 
  where table_schema != 'information_schema'
  and table_schema not like 'pg_%'
+ order by "column_index"
  {%- endcall -%}
  {# There's no point in returning anything as the jinja macro stuff calls #}
  {# str() on all returns. To get the results, you'll need to use #}
  {# context['load_result']('catalog') #}
 {%- endmacro %}
+
+
+{% macro snowflake__get_catalog() -%}
+ {%- call statement('catalog', fetch_result=True) -%}
+ with tables as (
+ select
+ table_schema as "table_schema",
+ table_name as "table_name",
+ table_type as "table_type"
+
+ from information_schema.tables
+ ),
+
+ columns as (
+
+ select
+ table_schema as "table_schema",
+ table_name as "table_name",
+ null as "table_comment",
+
+ column_name as "column_name",
+ ordinal_position as "column_index",
+ data_type as "column_type",
+ null as "column_comment"
+
+
+ from information_schema.columns
+
+ )
+
+ select *
+ from tables
+ join columns using ("table_schema", "table_name")
+ where "table_schema" != 'INFORMATION_SCHEMA'
+ order by "column_index"
+ {%- endcall -%}
+{%- endmacro %}

test/integration/029_docs_generate_tests/models/model.sql

@@ -0,0 +1,7 @@
+{{
+ config(
+ materialized='view'
+ )
+}}
+
+select * from {{ ref('seed') }}

test/integration/029_docs_generate_tests/seed/seed.csv

@@ -0,0 +1,2 @@
+id,first_name,email,ip_address,updated_at
+1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31