dbt ls command (#467)

core/dbt/logger.py

@@ -50,6 +50,11 @@ def notice(self, msg, *args, **kwargs):
 stdout_handler.setFormatter(logging.Formatter('%(message)s'))
 stdout_handler.setLevel(NOTICE)
 
+stderr_handler = logging.StreamHandler(sys.stderr)
+stderr_handler.setFormatter(logging.Formatter('%(message)s'))
+stderr_handler.setLevel(WARNING)
+
+
 logger = logging.getLogger('dbt')
 logger.addHandler(stdout_handler)
 logger.setLevel(DEBUG)
@@ -79,6 +84,21 @@ def notice(self, msg, *args, **kwargs):
 initialized = False
 
 
+def _swap_handler(logger, old, new):
+ if old in logger.handlers:
+ logger.handlers.remove(old)
+ if new not in logger.handlers:
+ logger.addHandler(new)
+
+
+def log_to_stderr(logger):
+ _swap_handler(logger, stdout_handler, stderr_handler)
+
+
+def log_to_stdout(logger):
+ _swap_handler(logger, stderr_handler, stdout_handler)
+
+
 def make_log_dir_if_missing(log_dir):
  import dbt.clients.system
  dbt.clients.system.make_directory(log_dir)
@@ -99,14 +119,17 @@ def default_formatter():
 
 
 def initialize_logger(debug_mode=False, path=None):
- global initialized, logger, stdout_handler
+ global initialized, logger, stdout_handler, stderr_handler
 
  if initialized:
  return
 
  if debug_mode:
+ # we'll only use one of these, but just set both up
  stdout_handler.setFormatter(default_formatter())
  stdout_handler.setLevel(DEBUG)
+ stderr_handler.setFormatter(default_formatter())
+ stderr_handler.setLevel(DEBUG)
 
  if path is not None:
  make_log_dir_if_missing(path)

core/dbt/main.py

@@ -22,6 +22,7 @@
 import dbt.task.serve as serve_task
 import dbt.task.freshness as freshness_task
 import dbt.task.run_operation as run_operation_task
+from dbt.task.list import ListTask
 from dbt.task.rpc_server import RPCServerTask
 from dbt.adapters.factory import reset_adapters
 
@@ -92,8 +93,8 @@ def main(args=None):
  exit_code = e.code
 
  except BaseException as e:
- logger.info("Encountered an error:")
- logger.info(str(e))
+ logger.warn("Encountered an error:")
+ logger.warn(str(e))
 
  if logger_initialized():
  logger.debug(traceback.format_exc())
@@ -183,6 +184,7 @@ def run_from_args(parsed):
  log_cache_events(getattr(parsed, 'log_cache_events', False))
  update_flags(parsed)
 
+ parsed.cls.pre_init_hook()
  logger.info("Running with dbt{}".format(dbt.version.installed))
 
  # this will convert DbtConfigErrors into RuntimeExceptions
@@ -569,6 +571,53 @@ def _build_rpc_subparser(subparsers, base_subparser):
  return sub
 
 
+def _build_list_subparser(subparsers, base_subparser):
+ sub = subparsers.add_parser(
+ 'list',
+ parents=[base_subparser],
+ help='list models'
+ )
+ sub.set_defaults(cls=ListTask, which='list')
+ resource_values = list(ListTask.ALL_RESOURCE_VALUES) + ['default', 'all']
+ sub.add_argument('--resource-type',
+ choices=resource_values,
+ action='append',
+ default=[],
+ dest='resource_types')
+ sub.add_argument('--output',
+ choices=['json', 'name', 'path', 'selector'],
+ default='selector')
+ sub.add_argument(
+ '-s',
+ '--select',
+ required=False,
+ nargs='+',
+ metavar='SELECTOR',
+ help="Specify the nodes to select.",
+ )
+ sub.add_argument(
+ '-m',
+ '--models',
+ required=False,
+ nargs='+',
+ metavar='SELECTOR',
+ help="Specify the models to select and set the resource-type to "
+ "'model'. Mutually exclusive with '--select' (or '-s') and "
+ "'--resource-type'",
+ )
+ sub.add_argument(
+ '--exclude',
+ required=False,
+ nargs='+',
+ metavar='SELECTOR',
+ help="Specify the models to exclude."
+ )
+ # in python 3.x you can use the 'aliases' kwarg, but in python 2.7 you get
+ # to do this
+ subparsers._name_parser_map['ls'] = sub
+ return sub
+
+
 def parse_args(args):
  p = DBTArgumentParser(
  prog='dbt: data build tool',
@@ -645,14 +694,15 @@ def parse_args(args):
 
  # make the subcommands that have their own subcommands
  docs_sub = _build_docs_subparser(subs, base_subparser)
- docs_subs = docs_sub.add_subparsers()
+ docs_subs = docs_sub.add_subparsers(title="Available sub-commands")
  source_sub = _build_source_subparser(subs, base_subparser)
- source_subs = source_sub.add_subparsers()
+ source_subs = source_sub.add_subparsers(title="Available sub-commands")
 
  _build_init_subparser(subs, base_subparser)
  _build_clean_subparser(subs, base_subparser)
  _build_debug_subparser(subs, base_subparser)
  _build_deps_subparser(subs, base_subparser)
+ _build_list_subparser(subs, base_subparser)
 
  archive_sub = _build_archive_subparser(subs, base_subparser)
  rpc_sub = _build_rpc_subparser(subs, base_subparser)

core/dbt/parser/base_sql.py

@@ -49,7 +49,7 @@ def load_and_parse(self, package_name, root_dir, relative_dirs,
 
  original_file_path = os.path.join(
  file_match.get('searched_path'),
- path)
+ file_match.get('relative_path'))
 
  result.append({
  'name': name,

core/dbt/task/base.py

@@ -46,6 +46,10 @@ def __init__(self, args, config):
  self.args = args
  self.config = config
 
+ @classmethod
+ def pre_init_hook(cls):
+ """A hook called before the task is initialized."""
+
  @classmethod
  def from_args(cls, args):
  try:

core/dbt/task/list.py

@@ -0,0 +1,140 @@
+from __future__ import print_function
+
+import json
+
+from dbt.task.runnable import GraphRunnableTask, ManifestTask
+from dbt.node_types import NodeType
+import dbt.exceptions
+from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.logger import log_to_stderr
+
+
+class ListTask(GraphRunnableTask):
+ DEFAULT_RESOURCE_VALUES = frozenset((
+ NodeType.Model,
+ NodeType.Archive,
+ NodeType.Seed,
+ NodeType.Test,
+ NodeType.Source,
+ ))
+ ALL_RESOURCE_VALUES = DEFAULT_RESOURCE_VALUES | frozenset((
+ NodeType.Analysis,
+ ))
+ ALLOWED_KEYS = frozenset((
+ 'alias',
+ 'name',
+ 'package_name',
+ 'depends_on',
+ 'tags',
+ 'config',
+ 'resource_type',
+ 'source_name',
+ ))
+
+ def __init__(self, args, config):
+ super(ListTask, self).__init__(args, config)
+ self.args.single_threaded = True
+ if self.args.models:
+ if self.args.select:
+ raise dbt.exceptions.RuntimeException(
+ '"models" and "select" are mutually exclusive arguments'
+ )
+ if self.args.resource_types:
+ raise dbt.exceptions.RuntimeException(
+ '"models" and "resource_type" are mutually exclusive '
+ 'arguments'
+ )
+
+ @classmethod
+ def pre_init_hook(cls):
+ """A hook called before the task is initialized."""
+ log_to_stderr(logger)
+
+ def _iterate_selected_nodes(self):
+ nodes = sorted(self.select_nodes())
+ if not nodes:
+ logger.warning('No nodes selected!')
+ return
+ for node in nodes:
+ yield self.manifest.nodes[node]
+
+ def generate_selectors(self):
+ for node in self._iterate_selected_nodes():
+ if node.resource_type == NodeType.Source:
+ yield 'source:{}'.format(node.unique_id)
+ else:
+ yield node.unique_id
+
+ def generate_names(self):
+ for node in self._iterate_selected_nodes():
+ if node.resource_type == NodeType.Source:
+ yield '{0.source_name}.{0.name}'.format(node)
+ else:
+ yield node.name
+
+ def generate_json(self):
+ for node in self._iterate_selected_nodes():
+ yield json.dumps({
+ k: v
+ for k, v in node.serialize().items()
+ if k in self.ALLOWED_KEYS
+ })
+
+ def generate_paths(self):
+ for node in self._iterate_selected_nodes():
+ yield node.get('original_file_path')
+
+ def run(self):
+ ManifestTask._runtime_initialize(self)
+ output = self.config.args.output
+ if output == 'selector':
+ generator = self.generate_selectors
+ elif output == 'name':
+ generator = self.generate_names
+ elif output == 'json':
+ generator = self.generate_json
+ elif output == 'path':
+ generator = self.generate_paths
+ else:
+ raise dbt.exceptions.IternalException(
+ 'Invalid output {}'.format(output)
+ )
+ for result in generator():
+ self.node_results.append(result)
+ print(result)
+ return self.node_results
+
+ @property
+ def resource_types(self):
+ if self.args.models:
+ return [NodeType.Model]
+
+ values = set(self.config.args.resource_types)
+ if not values:
+ return list(self.DEFAULT_RESOURCE_VALUES)
+
+ if 'default' in values:
+ values.remove('default')
+ values.update(self.DEFAULT_RESOURCE_VALUES)
+ if 'all' in values:
+ values.remove('all')
+ values.update(self.ALL_RESOURCE_VALUES)
+ return list(values)
+
+ @property
+ def selector(self):
+ if self.args.models:
+ return self.args.models
+ else:
+ return self.args.select
+
+ def build_query(self):
+ return {
+ "include": self.selector,
+ "exclude": self.args.exclude,
+ "resource_types": self.resource_types,
+ "tags": [],
+ }
+
+ def interpret_results(self, results):
+ return bool(results)

test/integration/047_dbt_ls_test/analyses/analysis.sql

@@ -0,0 +1 @@
+select 4 as id

test/integration/047_dbt_ls_test/archives/archive.sql

@@ -0,0 +1,12 @@
+{% archive my_archive %}
+ {{
+ config(
+ target_database=var('target_database', database),
+ target_schema=schema,
+ unique_key='id',
+ strategy='timestamp',
+ updated_at='updated_at',
+ )
+ }}
+ select * from {{database}}.{{schema}}.seed
+{% endarchive %}

test/integration/047_dbt_ls_test/data/seed.csv

@@ -0,0 +1,2 @@
+a,b
+1,2

test/integration/047_dbt_ls_test/macros/macro_stuff.sql

@@ -0,0 +1,7 @@
+{% macro cool_macro() %}
+ wow!
+{% endmacro %}
+
+{% macro other_cool_macro(a, b) %}
+ cool!
+{% endmacro %}

test/integration/047_dbt_ls_test/models/docs.md

@@ -0,0 +1,3 @@
+{% docs my_docs %}
+ some docs
+{% enddocs %}

test/integration/047_dbt_ls_test/models/outer.sql

@@ -0,0 +1 @@
+select 1 as id

test/integration/047_dbt_ls_test/models/schema.yml

@@ -0,0 +1,15 @@
+version: 2
+models:
+ - name: outer
+ description: The outer table
+ columns:
+ - name: id
+ description: The id value
+ tests:
+ - unique
+ - not_null
+
+sources:
+ - name: my_source
+ tables:
+ - name: my_table

test/integration/047_dbt_ls_test/models/sub/inner.sql

@@ -0,0 +1 @@
+select * from {{ ref('outer') }}