Make seeds ref'able, provide for seed configuration

dbt/adapters/bigquery.py

@@ -403,7 +403,7 @@ def warning_on_hooks(cls, hook_type):
 
  @classmethod
  def add_query(cls, profile, sql, model_name=None, auto_begin=True,
- bindings=None):
+ bindings=None, abridge_sql_log=False):
  if model_name in ['on-run-start', 'on-run-end']:
  cls.warning_on_hooks(model_name)
  else:

dbt/adapters/default.py

@@ -524,7 +524,7 @@ def close(cls, connection):
 
  @classmethod
  def add_query(cls, profile, sql, model_name=None, auto_begin=True,
- bindings=None):
+ bindings=None, abridge_sql_log=False):
  connection = cls.get_connection(profile, model_name)
  connection_name = connection.get('name')
 
@@ -535,7 +535,10 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True,
  .format(cls.type(), connection_name))
 
  with cls.exception_handler(profile, sql, model_name, connection_name):
- logger.debug('On %s: %s', connection_name, sql)
+ if abridge_sql_log:
+ logger.debug('On %s: %s....', connection_name, sql[0:512])
+ else:
+ logger.debug('On %s: %s', connection_name, sql)
  pre = time.time()
 
  cursor = connection.get('handle').cursor()

dbt/adapters/postgres.py

@@ -235,4 +235,4 @@ def load_csv_rows(cls, profile, schema, table_name, agate_table):
  cols_sql,
  ",\n".join(placeholders)))
 
- cls.add_query(profile, sql, bindings=bindings)
+ cls.add_query(profile, sql, bindings=bindings, abridge_sql_log=True)

dbt/adapters/snowflake.py

@@ -181,7 +181,7 @@ def check_schema_exists(cls, profile, schema, model_name=None):
 
  @classmethod
  def add_query(cls, profile, sql, model_name=None, auto_begin=True,
- select_schema=True, bindings=None):
+ select_schema=True, bindings=None, abridge_sql_log=False):
  # snowflake only allows one query per api call.
  queries = sql.strip().split(";")
  cursor = None
@@ -211,7 +211,7 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True,
 
  connection, cursor = super(PostgresAdapter, cls).add_query(
  profile, individual_query, model_name, auto_begin,
- bindings=bindings)
+ bindings=bindings, abridge_sql_log=abridge_sql_log)
 
  return connection, cursor
 

dbt/loader.py

@@ -192,7 +192,8 @@ def load_project(cls, root_project, all_projects, project, project_name,
  all_projects=all_projects,
  root_dir=project.get('project-root'),
  relative_dirs=project.get('data-paths', []),
- resource_type=NodeType.Seed)
+ resource_type=NodeType.Seed,
+ macros=macros)
 
 
 # node loaders

dbt/model.py

@@ -5,6 +5,7 @@
 from dbt.compat import basestring
 
 from dbt.utils import split_path, deep_merge, DBTConfigKeys
+from dbt.node_types import NodeType
 
 
 class SourceConfig(object):
@@ -24,11 +25,12 @@ class SourceConfig(object):
  'bind'
  ]
 
- def __init__(self, active_project, own_project, fqn):
+ def __init__(self, active_project, own_project, fqn, node_type):
  self._config = None
  self.active_project = active_project
  self.own_project = own_project
  self.fqn = fqn
+ self.node_type = node_type
 
  # the config options defined within the model
  self.in_model_config = {}
@@ -133,7 +135,10 @@ def get_project_config(self, project):
  for k in SourceConfig.ExtendDictFields:
  config[k] = {}
 
- model_configs = project.get('models')
+ if self.node_type == NodeType.Seed:
+ model_configs = project.get('seeds')
+ else:
+ model_configs = project.get('models')
 
  if model_configs is None:
  return config

dbt/node_runners.py

@@ -79,16 +79,16 @@ def raise_on_first_error(self):
  return False
 
  @classmethod
- def is_model(cls, node):
- return node.get('resource_type') == NodeType.Model
+ def is_refable(cls, node):
+ return node.get('resource_type') in [NodeType.Model, NodeType.Seed]
 
  @classmethod
  def is_ephemeral(cls, node):
  return dbt.utils.get_materialization(node) == 'ephemeral'
 
  @classmethod
  def is_ephemeral_model(cls, node):
- return cls.is_model(node) and cls.is_ephemeral(node)
+ return cls.is_refable(node) and cls.is_ephemeral(node)
 
  def safe_run(self, flat_graph, existing):
  catchable_errors = (dbt.exceptions.CompilationException,
@@ -175,7 +175,7 @@ def do_skip(self):
  def get_model_schemas(cls, flat_graph):
  schemas = set()
  for node in flat_graph['nodes'].values():
- if cls.is_model(node) and not cls.is_ephemeral(node):
+ if cls.is_refable(node) and not cls.is_ephemeral(node):
  schemas.add(node['schema'])
 
  return schemas

dbt/parser.py

@@ -39,15 +39,15 @@ def resolve_ref(flat_graph, target_model_name, target_model_package,
  current_project, node_package):
 
  if target_model_package is not None:
- return dbt.utils.find_model_by_name(
+ return dbt.utils.find_refable_by_name(
  flat_graph,
  target_model_name,
  target_model_package)
 
  target_model = None
 
  # first pass: look for models in the current_project
- target_model = dbt.utils.find_model_by_name(
+ target_model = dbt.utils.find_refable_by_name(
  flat_graph,
  target_model_name,
  current_project)
@@ -56,7 +56,7 @@ def resolve_ref(flat_graph, target_model_name, target_model_package,
  return target_model
 
  # second pass: look for models in the node's package
- target_model = dbt.utils.find_model_by_name(
+ target_model = dbt.utils.find_refable_by_name(
  flat_graph,
  target_model_name,
  node_package)
@@ -67,7 +67,7 @@ def resolve_ref(flat_graph, target_model_name, target_model_package,
  # final pass: look for models in any package
  # todo: exclude the packages we have already searched. overriding
  # a package model in another package doesn't necessarily work atm
- return dbt.utils.find_model_by_name(
+ return dbt.utils.find_refable_by_name(
  flat_graph,
  target_model_name,
  None)
@@ -201,7 +201,10 @@ def parse_node(node, node_path, root_project_config, package_project_config,
  fqn = get_fqn(node.get('path'), package_project_config, fqn_extra)
 
  config = dbt.model.SourceConfig(
- root_project_config, package_project_config, fqn)
+ root_project_config,
+ package_project_config,
+ fqn,
+ node['resource_type'])
 
  node['unique_id'] = node_path
  node['empty'] = ('raw_sql' in node and len(node['raw_sql'].strip()) == 0)

dbt/utils.py

@@ -125,14 +125,14 @@ def model_immediate_name(model, non_destructive):
  return "{}__dbt_tmp".format(model_name)
 
 
-def find_model_by_name(flat_graph, target_name, target_package):
+def find_refable_by_name(flat_graph, target_name, target_package):
  return find_by_name(flat_graph, target_name, target_package,
- 'nodes', NodeType.Model)
+ 'nodes', [NodeType.Model, NodeType.Seed])
 
 
 def find_macro_by_name(flat_graph, target_name, target_package):
  return find_by_name(flat_graph, target_name, target_package,
- 'macros', NodeType.Macro)
+ 'macros', [NodeType.Macro])
 
 
 def find_by_name(flat_graph, target_name, target_package, subgraph,
@@ -146,7 +146,7 @@ def find_by_name(flat_graph, target_name, target_package, subgraph,
 
  resource_type, package_name, node_name = node_parts
 
- if (resource_type == nodetype and
+ if (resource_type in nodetype and
  ((target_name == node_name) and
  (target_package is None or
  target_package == package_name))):

test/integration/005_simple_seed_test/data-config/seed_disabled.csv

@@ -0,0 +1,21 @@
+id,first_name,email,ip_address,birthday
+1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31
+2,Larry,lperkins1@toplist.cz,64.210.133.162,1978-05-09 04:15:14
+3,Anna,amontgomery2@miitbeian.gov.cn,168.104.64.114,2011-10-16 04:07:57
+4,Sandra,sgeorge3@livejournal.com,229.235.252.98,1973-07-19 10:52:43
+5,Fred,fwoods4@google.cn,78.229.170.124,2012-09-30 16:38:29
+6,Stephen,shanson5@livejournal.com,182.227.157.105,1995-11-07 21:40:50
+7,William,wmartinez6@upenn.edu,135.139.249.50,1982-09-05 03:11:59
+8,Jessica,jlong7@hao123.com,203.62.178.210,1991-10-16 11:03:15
+9,Douglas,dwhite8@tamu.edu,178.187.247.1,1979-10-01 09:49:48
+10,Lisa,lcoleman9@nydailynews.com,168.234.128.249,2011-05-26 07:45:49
+11,Ralph,rfieldsa@home.pl,55.152.163.149,1972-11-18 19:06:11
+12,Louise,lnicholsb@samsung.com,141.116.153.154,2014-11-25 20:56:14
+13,Clarence,cduncanc@sfgate.com,81.171.31.133,2011-11-17 07:02:36
+14,Daniel,dfranklind@omniture.com,8.204.211.37,1980-09-13 00:09:04
+15,Katherine,klanee@auda.org.au,176.96.134.59,1997-08-22 19:36:56
+16,Billy,bwardf@wikia.com,214.108.78.85,2003-10-19 02:14:47
+17,Annie,agarzag@ocn.ne.jp,190.108.42.70,1988-10-28 15:12:35
+18,Shirley,scolemanh@fastcompany.com,109.251.164.84,1988-08-24 10:50:57
+19,Roger,rfrazieri@scribd.com,38.145.218.108,1985-12-31 15:17:15
+20,Lillian,lstanleyj@goodreads.com,47.57.236.17,1970-06-08 02:09:05

test/integration/005_simple_seed_test/data-config/seed_enabled.csv

@@ -0,0 +1,21 @@
+id,first_name,email,ip_address,birthday
+1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31
+2,Larry,lperkins1@toplist.cz,64.210.133.162,1978-05-09 04:15:14
+3,Anna,amontgomery2@miitbeian.gov.cn,168.104.64.114,2011-10-16 04:07:57
+4,Sandra,sgeorge3@livejournal.com,229.235.252.98,1973-07-19 10:52:43
+5,Fred,fwoods4@google.cn,78.229.170.124,2012-09-30 16:38:29
+6,Stephen,shanson5@livejournal.com,182.227.157.105,1995-11-07 21:40:50
+7,William,wmartinez6@upenn.edu,135.139.249.50,1982-09-05 03:11:59
+8,Jessica,jlong7@hao123.com,203.62.178.210,1991-10-16 11:03:15
+9,Douglas,dwhite8@tamu.edu,178.187.247.1,1979-10-01 09:49:48
+10,Lisa,lcoleman9@nydailynews.com,168.234.128.249,2011-05-26 07:45:49
+11,Ralph,rfieldsa@home.pl,55.152.163.149,1972-11-18 19:06:11
+12,Louise,lnicholsb@samsung.com,141.116.153.154,2014-11-25 20:56:14
+13,Clarence,cduncanc@sfgate.com,81.171.31.133,2011-11-17 07:02:36
+14,Daniel,dfranklind@omniture.com,8.204.211.37,1980-09-13 00:09:04
+15,Katherine,klanee@auda.org.au,176.96.134.59,1997-08-22 19:36:56
+16,Billy,bwardf@wikia.com,214.108.78.85,2003-10-19 02:14:47
+17,Annie,agarzag@ocn.ne.jp,190.108.42.70,1988-10-28 15:12:35
+18,Shirley,scolemanh@fastcompany.com,109.251.164.84,1988-08-24 10:50:57
+19,Roger,rfrazieri@scribd.com,38.145.218.108,1985-12-31 15:17:15
+20,Lillian,lstanleyj@goodreads.com,47.57.236.17,1970-06-08 02:09:05

test/integration/005_simple_seed_test/test_simple_seed.py

@@ -27,7 +27,7 @@ def test_simple_seed(self):
  self.run_dbt(["seed"])
  self.assertTablesEqual("seed_actual","seed_expected")
 
- # this should truncate the seed_actual table, the re-insert
+ # this should truncate the seed_actual table, then re-insert
  self.run_dbt(["seed"])
  self.assertTablesEqual("seed_actual","seed_expected")
 
@@ -40,3 +40,84 @@ def test_simple_seed_with_drop(self):
  # this should drop the seed table, then re-create
  self.run_dbt(["seed", "--drop-existing"])
  self.assertTablesEqual("seed_actual","seed_expected")
+
+
+class TestSimpleSeedCustomSchema(DBTIntegrationTest):
+
+ def setUp(self):
+ DBTIntegrationTest.setUp(self)
+ self.run_sql_file("test/integration/005_simple_seed_test/seed.sql")
+
+ @property
+ def schema(self):
+ return "simple_seed_005"
+
+ @property
+ def models(self):
+ return "test/integration/005_simple_seed_test/models"
+
+ @property
+ def project_config(self):
+ return {
+ "data-paths": ['test/integration/005_simple_seed_test/data'],
+ "seeds": {
+ "schema": "custom_schema"
+ }
+ }
+
+ @attr(type='postgres')
+ def test_simple_seed_with_schema(self):
+ schema_name = "{}_{}".format(self.unique_schema(), 'custom_schema')
+
+ self.run_dbt(["seed"])
+ self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name)
+
+ # this should truncate the seed_actual table, then re-insert
+ self.run_dbt(["seed"])
+ self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name)
+
+
+ @attr(type='postgres')
+ def test_simple_seed_with_drop_and_schema(self):
+ schema_name = "{}_{}".format(self.unique_schema(), 'custom_schema')
+
+ self.run_dbt(["seed"])
+ self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name)
+
+ # this should drop the seed table, then re-create
+ self.run_dbt(["seed", "--full-refresh"])
+ self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name)
+
+
+class TestSimpleSeedDisabled(DBTIntegrationTest):
+
+ @property
+ def schema(self):
+ return "simple_seed_005"
+
+ @property
+ def models(self):
+ return "test/integration/005_simple_seed_test/models"
+
+ @property
+ def project_config(self):
+ return {
+ "data-paths": ['test/integration/005_simple_seed_test/data-config'],
+ "seeds": {
+ "test": {
+ "seed_enabled": {
+ "enabled": True
+ },
+ "seed_disabled": {
+ "enabled": False
+ }
+ }
+ }
+ }
+
+ @attr(type='postgres')
+ def test_simple_seed_with_disabled(self):
+ self.run_dbt(["seed"])
+ self.assertTableDoesExist('seed_enabled')
+ self.assertTableDoesNotExist('seed_disabled')
+