Merge pull request #880 from fishtown-analytics/new-schema-yaml-syntax

Support new schema.yml syntax (#790)

CHANGELOG.md

@@ -1,3 +1,11 @@
+## dbt 0.11.0 (unreleased, codename: Isaac Asimov)
+
+### Features
+
+- Extend catalog and manifest to also support Snowflake, BigQuery, and Redshift
+- Add a 'generated_at' field to both the manifest and the catalog.
+- Version 2 of schema.yml, which allows users to create table and column comments that end up in the manifest
+
 ## dbt 0.10.2 (unreleased, codename: Betsy Ross)
 
 ### Overview

dbt/api/object.py

@@ -36,6 +36,12 @@ def __init__(self, **kwargs):
  self._contents = deep_merge(self.DEFAULTS, kwargs)
  self.validate()
 
+ def __str__(self):
+ return '{}(**{})'.format(self.__class__.__name__, self._contents)
+
+ def __repr__(self):
+ return '{}(**{})'.format(self.__class__.__name__, self._contents)
+
  def incorporate(self, **kwargs):
  """
  Given a list of kwargs, incorporate these arguments

dbt/contracts/graph/parsed.py

@@ -1,6 +1,8 @@
 from dbt.api import APIObject
 from dbt.utils import deep_merge, timestring
 from dbt.node_types import NodeType
+from dbt.exceptions import raise_duplicate_resource_name, \
+ raise_patch_targets_not_found
 
 import dbt.clients.jinja
 
@@ -66,6 +68,25 @@
 }
 
 
+# Note that description must be present, but may be empty.
+COLUMN_INFO_CONTRACT = {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'description': 'Information about a single column in a model',
+ 'properties': {
+ 'name': {
+ 'type': 'string',
+ 'description': 'The column name',
+ },
+ 'description': {
+ 'type': 'string',
+ 'description': 'A description of the column',
+ },
+ },
+ 'required': ['name', 'description'],
+}
+
+
 PARSED_NODE_CONTRACT = deep_merge(
  UNPARSED_NODE_CONTRACT,
  {
@@ -151,6 +172,20 @@
  'type': 'string',
  }
  },
+ 'description': {
+ 'type': 'string',
+ 'description': 'A user-supplied description of the model',
+ },
+ 'columns': {
+ 'type': 'array',
+ 'items': COLUMN_INFO_CONTRACT,
+ },
+ 'patch_path': {
+ 'type': 'string',
+ 'description': (
+ 'The path to the patch source if the node was patched'
+ ),
+ },
  },
  'required': UNPARSED_NODE_CONTRACT['required'] + [
  'unique_id', 'fqn', 'schema', 'refs', 'depends_on', 'empty',
@@ -160,6 +195,42 @@
 )
 
 
+# The parsed node update is only the 'patch', not the test. The test became a
+# regular parsed node. Note that description and columns must be present, but
+# may be empty.
+PARSED_NODE_PATCH_CONTRACT = {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'description': 'A collection of values that can be set on a node',
+ 'properties': {
+ 'name': {
+ 'type': 'string',
+ 'description': 'The name of the node this modifies',
+ },
+ 'description': {
+ 'type': 'string',
+ 'description': 'The description of the node to add',
+ },
+ 'original_file_path': {
+ 'type': 'string',
+ 'description': (
+ 'Relative path to the originating file path for the patch '
+ 'from the project root'
+ ),
+ },
+ 'columns': {
+ 'type': 'array',
+ 'items': COLUMN_INFO_CONTRACT,
+ }
+ },
+ 'required': ['name', 'original_file_path', 'description', 'columns'],
+}
+
+
+class ParsedNodePatch(APIObject):
+ SCHEMA = PARSED_NODE_PATCH_CONTRACT
+
+
 PARSED_NODES_CONTRACT = {
  'type': 'object',
  'additionalProperties': False,
@@ -308,6 +379,18 @@ def to_dict(self):
  ret['agate_table'] = self.agate_table
  return ret
 
+ def patch(self, patch):
+ """Given a ParsedNodePatch, add the new information to the node."""
+ # explicitly pick out the parts to update so we don't inadvertently
+ # step on the model name or anything
+ self._contents.update({
+ 'patch_path': patch.original_file_path,
+ 'description': patch.description,
+ 'columns': patch.columns,
+ })
+ # patches always trigger re-validation
+ self.validate()
+
 
 class ParsedMacro(APIObject):
  SCHEMA = PARSED_MACRO_CONTRACT
@@ -405,6 +488,39 @@ def find_operation_by_name(self, name, package):
  return self._find_by_name(name, package, 'macros',
  [NodeType.Operation])
 
+ def add_nodes(self, new_nodes):
+ """Add the given dict of new nodes to the manifest."""
+ for unique_id, node in new_nodes.items():
+ if unique_id in self.nodes:
+ raise_duplicate_resource_name(node, self.nodes[unique_id])
+ self.nodes[unique_id] = node
+
+ def patch_nodes(self, patches):
+ """Patch nodes with the given dict of patches. Note that this consumes
+ the input!
+ """
+ # because we don't have any mapping from node _names_ to nodes, and we
+ # only have the node name in the patch, we have to iterate over all the
+ # nodes looking for matching names. We could use _find_by_name if we
+ # were ok with doing an O(n*m) search (one nodes scan per patch)
+ for node in self.nodes.values():
+ if node.resource_type != NodeType.Model:
+ continue
+ patch = patches.pop(node.name, None)
+ if not patch:
+ continue
+ node.patch(patch)
+
+ # log debug-level warning about nodes we couldn't find
+ if patches:
+ for patch in patches.values():
+ # since patches aren't nodes, we can't use the existing
+ # target_not_found warning
+ logger.debug((
+ 'WARNING: Found documentation for model "{}" which was '
+ 'not found or is disabled').format(patch.name)
+ )
+
  def to_flat_graph(self):
  """Convert the parsed manifest to the 'flat graph' that the compiler
  expects.

dbt/contracts/graph/unparsed.py

@@ -85,3 +85,79 @@ class UnparsedMacro(APIObject):
 
 class UnparsedNode(APIObject):
  SCHEMA = UNPARSED_NODE_CONTRACT
+
+
+COLUMN_TEST_CONTRACT = {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ 'name': {
+ 'type': 'string',
+ 'description': 'The name of the column this test is for',
+ },
+ 'description': {
+ 'type': 'string',
+ 'description': 'The description of this test',
+ },
+ 'tests': {
+ 'type': 'array',
+ 'items': {
+ "anyOf": [
+ # 'not_null', 'unique', ...
+ {'type': 'string'},
+ # 'relationships: {...}', 'accepted_values: {...}'
+ {'type': 'object', 'additionalProperties': True}
+ ],
+ },
+ 'description': 'The list of tests to perform',
+ },
+ },
+ 'required': ['name'],
+}
+
+
+UNPARSED_NODE_UPDATE_CONTRACT = {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'description': (
+ 'A collection of the unparsed node updates, as provided in the '
+ '"models" section of schema.yml'
+ ),
+ 'properties': {
+ 'name': {
+ 'type': 'string',
+ 'description': (
+ 'The name of this node, which is the name of the model it'
+ 'refers to'
+ ),
+ 'minLength': 1,
+ },
+ 'description': {
+ 'type': 'string',
+ 'description': (
+ 'The raw string description of the node after parsing the yaml'
+ ),
+ },
+ 'columns': {
+ 'type': 'array',
+ 'items': COLUMN_TEST_CONTRACT,
+ },
+ 'tests': {
+ 'type': 'array',
+ 'items': {
+ "anyOf": [
+ {'type': 'string'},
+ {'type': 'object', 'additionalProperties': True}
+ ],
+ },
+ },
+ },
+ 'required': ['name'],
+}
+
+
+class UnparsedNodeUpdate(APIObject):
+ """An unparsed node update is the blueprint for tests to be added and nodes
+ to be updated, referencing a certain node (specifically, a Model).
+ """
+ SCHEMA = UNPARSED_NODE_UPDATE_CONTRACT

dbt/exceptions.py

@@ -357,3 +357,27 @@ def raise_ambiguous_alias(node_1, node_2):
  duped_name,
  node_1['unique_id'], node_1['original_file_path'],
  node_2['unique_id'], node_2['original_file_path']))
+
+
+def raise_patch_targets_not_found(patches):
+ patch_list = '\n\t'.join(
+ 'model {} (referenced in path {})'.format(p.name, p.original_file_path)
+ for p in patches.values()
+ )
+ raise_compiler_error(
+ 'dbt could not find models for the following patches:\n\t{}'.format(
+ patch_list
+ )
+ )
+
+
+def raise_duplicate_patch_name(name, patch_1, patch_2):
+ raise_compiler_error(
+ 'dbt found two schema.yml entries for the same model named {}. The '
+ 'first patch was specified in {} and the second in {}. Models and '
+ 'their associated columns may only be described a single time.'.format(
+ name,
+ patch_1,
+ patch_2,
+ )
+ )

dbt/include/global_project/macros/schema_tests/accepted_values.sql

@@ -1,10 +1,12 @@
 
-{% macro test_accepted_values(model, field, values) %}
+{% macro test_accepted_values(model, values) %}
+
+{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}
 
 with all_values as (
 
  select distinct
- {{ field }} as value_field
+ {{ column_name }} as value_field
 
  from {{ model }}
 

dbt/include/global_project/macros/schema_tests/not_null.sql

@@ -1,9 +1,11 @@
 
-{% macro test_not_null(model, arg) %}
+{% macro test_not_null(model) %}
+
+{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
 
 select count(*)
 from {{ model }}
-where {{ arg }} is null
+where {{ column_name }} is null
 
 {% endmacro %}
 

dbt/include/global_project/macros/schema_tests/relationships.sql

@@ -1,6 +1,7 @@
 
-{% macro test_relationships(model, field, to, from) %}
+{% macro test_relationships(model, to, from) %}
 
+{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}
 
 select count(*)
 from (
@@ -10,7 +11,7 @@ from (
 
  from {{ model }}
  where {{ from }} is not null
- and {{ from }} not in (select {{ field }}
+ and {{ from }} not in (select {{ column_name }}
  from {{ to }})
 
 ) validation_errors

dbt/include/global_project/macros/schema_tests/unique.sql

@@ -1,15 +1,17 @@
 
-{% macro test_unique(model, arg) %}
+{% macro test_unique(model) %}
+
+{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
 
 select count(*)
 from (
 
  select
- {{ arg }}
+ {{ column_name }}
 
  from {{ model }}
- where {{ arg }} is not null
- group by {{ arg }}
+ where {{ column_name }} is not null
+ group by {{ column_name }}
  having count(*) > 1
 
 ) validation_errors