From 88dda8b34c3e36de5f057433c3269a0018a126d8 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 14 Mar 2017 18:08:40 -0400 Subject: [PATCH 01/10] shim out macros, change the flat graph to contain nodes and macros --- dbt/compilation.py | 165 ++++++----- dbt/contracts/graph/compiled.py | 35 ++- dbt/contracts/graph/parsed.py | 69 +++-- dbt/contracts/graph/unparsed.py | 22 +- dbt/model.py | 1 + dbt/parser.py | 123 +++++--- dbt/utils.py | 4 +- test/unit/test_compiler.py | 490 ++++++++++++++++++-------------- test/unit/test_graph.py | 2 +- test/unit/test_parser.py | 217 +++++++++++--- 10 files changed, 707 insertions(+), 421 deletions(-) diff --git a/dbt/compilation.py b/dbt/compilation.py index 23c42ebe68a..d298995d2bb 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -1,6 +1,5 @@ import os -from collections import defaultdict, OrderedDict -import time +from collections import OrderedDict import sqlparse import dbt.project @@ -8,8 +7,7 @@ from dbt.model import Model, NodeType from dbt.source import Source -from dbt.utils import find_model_by_fqn, find_model_by_name, \ - split_path, This, Var, is_enabled, get_materialization +from dbt.utils import This, Var, is_enabled, get_materialization from dbt.linker import Linker from dbt.runtime import RuntimeContext @@ -57,27 +55,27 @@ def compile_and_print_status(project, args): logger.info("Compiled {}".format(stat_line)) -def prepend_ctes(model, all_models): - model, _, all_models = recursively_prepend_ctes(model, all_models) +def prepend_ctes(model, flat_graph): + model, _, flat_graph = recursively_prepend_ctes(model, flat_graph) - return (model, all_models) + return (model, flat_graph) -def recursively_prepend_ctes(model, all_models): +def recursively_prepend_ctes(model, flat_graph): if dbt.flags.STRICT_MODE: - dbt.contracts.graph.compiled.validate_one(model) - dbt.contracts.graph.compiled.validate(all_models) + dbt.contracts.graph.compiled.validate_node(model) + dbt.contracts.graph.compiled.validate(flat_graph) model = model.copy() prepend_ctes = OrderedDict() if model.get('all_ctes_injected') is True: - return (model, model.get('extra_ctes').keys(), all_models) + return (model, model.get('extra_ctes').keys(), flat_graph) for cte_id in model.get('extra_ctes').keys(): - cte_to_add = all_models.get(cte_id) - cte_to_add, new_prepend_ctes, all_models = recursively_prepend_ctes( - cte_to_add, all_models) + cte_to_add = flat_graph.get('nodes').get(cte_id) + cte_to_add, new_prepend_ctes, flat_graph = recursively_prepend_ctes( + cte_to_add, flat_graph) prepend_ctes.update(new_prepend_ctes) new_cte_name = '__dbt__CTE__{}'.format(cte_to_add.get('name')) @@ -91,9 +89,9 @@ def recursively_prepend_ctes(model, all_models): model.get('compiled_sql'), model.get('extra_ctes')) - all_models[model.get('unique_id')] = model + flat_graph['nodes'][model.get('unique_id')] = model - return (model, prepend_ctes, all_models) + return (model, prepend_ctes, flat_graph) def inject_ctes_into_sql(sql, ctes): @@ -208,8 +206,8 @@ def do_ref(*args): target_model_id = target_model.get('unique_id') - if target_model_id not in model['depends_on']: - model['depends_on'].append(target_model_id) + if target_model_id not in model['depends_on']['nodes']: + model['depends_on']['nodes'].append(target_model_id) if get_materialization(target_model) == 'ephemeral': model['extra_ctes'][target_model_id] = None @@ -230,8 +228,7 @@ def wrapped_do_ref(*args): return wrapped_do_ref - def get_compiler_context(self, linker, model, models, - macro_generator=None): + def get_compiler_context(self, linker, model, models): context = self.project.context() adapter = get_adapter(self.project.run_environment()) @@ -252,6 +249,7 @@ def get_compiler_context(self, linker, model, models, context['invocation_id'] = '{{ invocation_id }}' context['sql_now'] = adapter.date_function + macro_generator = None if macro_generator is not None: for macro_data in macro_generator(context): macro = macro_data["macro"] @@ -296,7 +294,7 @@ def get_context(self, linker, model, models): return runtime - def compile_node(self, linker, node, nodes, macro_generator): + def compile_node(self, linker, node, flat_graph): logger.debug("Compiling {}".format(node.get('unique_id'))) compiled_node = node.copy() @@ -308,8 +306,7 @@ def compile_node(self, linker, node, nodes, macro_generator): 'injected_sql': None, }) - context = self.get_compiler_context(linker, compiled_node, nodes, - macro_generator) + context = self.get_compiler_context(linker, compiled_node, flat_graph) compiled_node['compiled_sql'] = dbt.clients.jinja.get_rendered( node.get('raw_sql'), @@ -325,29 +322,38 @@ def write_graph_file(self, linker): graph_path = os.path.join(self.project['target-path'], filename) linker.write_graph(graph_path) - def compile_nodes(self, linker, nodes, macro_generator): + def compile_graph(self, linker, flat_graph): all_projects = self.get_all_projects() - compiled_nodes = {} - injected_nodes = {} - wrapped_nodes = {} + compiled_graph = { + 'nodes': {}, + 'macros': {}, + } + injected_graph = { + 'nodes': {}, + 'macros': {}, + } + wrapped_graph = { + 'nodes': {}, + 'macros': {}, + } written_nodes = [] - for name, node in nodes.items(): - compiled_nodes[name] = self.compile_node(linker, node, nodes, - macro_generator) + for name, node in flat_graph.get('nodes').items(): + compiled_graph['nodes'][name] = \ + self.compile_node(linker, node, flat_graph) if dbt.flags.STRICT_MODE: - dbt.contracts.graph.compiled.validate(compiled_nodes) + dbt.contracts.graph.compiled.validate(compiled_graph) - for name, node in compiled_nodes.items(): - node, compiled_nodes = prepend_ctes(node, compiled_nodes) - injected_nodes[name] = node + for name, node in compiled_graph.get('nodes').items(): + node, compiled_graph = prepend_ctes(node, compiled_graph) + injected_graph['nodes'][name] = node if dbt.flags.STRICT_MODE: - dbt.contracts.graph.compiled.validate(injected_nodes) + dbt.contracts.graph.compiled.validate(injected_graph) - for name, injected_node in injected_nodes.items(): + for name, injected_node in injected_graph.get('nodes').items(): # now turn model nodes back into the old-style model object for # wrapping if injected_node.get('resource_type') in [NodeType.Test, @@ -364,7 +370,7 @@ def compile_nodes(self, linker, nodes, macro_generator): injected_node['wrapped_sql'] = injected_node.get( 'injected_sql') - wrapped_nodes[name] = injected_node + wrapped_graph['nodes'][name] = injected_node elif injected_node.get('resource_type') == NodeType.Archive: # unfortunately we do everything automagically for @@ -382,13 +388,15 @@ def compile_nodes(self, linker, nodes, macro_generator): cfg = injected_node.get('config', {}) model._config = cfg - context = self.get_context(linker, model, injected_nodes) + context = self.get_context(linker, + model, + injected_graph.get('nodes')) wrapped_stmt = model.compile( injected_node.get('injected_sql'), self.project, context) injected_node['wrapped_sql'] = wrapped_stmt - wrapped_nodes[name] = injected_node + wrapped_graph['nodes'][name] = injected_node build_path = os.path.join('build', injected_node.get('path')) @@ -401,17 +409,18 @@ def compile_nodes(self, linker, nodes, macro_generator): injected_node['build_path'] = build_path linker.add_node(injected_node.get('unique_id')) - project = all_projects[injected_node.get('package_name')] linker.update_node_data( injected_node.get('unique_id'), injected_node) - for dependency in injected_node.get('depends_on'): - if compiled_nodes.get(dependency): + for dependency in injected_node['depends_on']['nodes']: + if compiled_graph.get('nodes').get(dependency): linker.dependency( injected_node.get('unique_id'), - compiled_nodes.get(dependency).get('unique_id')) + (compiled_graph.get('nodes') + .get(dependency) + .get('unique_id'))) else: dbt.exceptions.dependency_not_found(model, dependency) @@ -420,7 +429,7 @@ def compile_nodes(self, linker, nodes, macro_generator): if cycle: raise RuntimeError("Found a cycle: {}".format(cycle)) - return wrapped_nodes, written_nodes + return wrapped_graph, written_nodes def generate_macros(self, all_macros): def do_gen(ctx): @@ -445,7 +454,22 @@ def get_all_projects(self): return all_projects - def get_parsed_models(self, root_project, all_projects, macro_generator): + def get_parsed_macros(self, root_project, all_projects): + parsed_macros = {} + + for name, project in all_projects.items(): + parsed_macros.update( + dbt.parser.load_and_parse_macros( + package_name=name, + root_project=root_project, + all_projects=all_projects, + root_dir=project.get('project-root'), + relative_dirs=project.get('macro-paths', []), + resource_type=NodeType.Macro)) + + return parsed_macros + + def get_parsed_models(self, root_project, all_projects): parsed_models = {} for name, project in all_projects.items(): @@ -456,12 +480,11 @@ def get_parsed_models(self, root_project, all_projects, macro_generator): all_projects=all_projects, root_dir=project.get('project-root'), relative_dirs=project.get('source-paths', []), - resource_type=NodeType.Model, - macro_generator=macro_generator)) + resource_type=NodeType.Model)) return parsed_models - def get_parsed_analyses(self, root_project, all_projects, macro_generator): + def get_parsed_analyses(self, root_project, all_projects): parsed_models = {} for name, project in all_projects.items(): @@ -472,13 +495,11 @@ def get_parsed_analyses(self, root_project, all_projects, macro_generator): all_projects=all_projects, root_dir=project.get('project-root'), relative_dirs=project.get('analysis-paths', []), - resource_type=NodeType.Analysis, - macro_generator=macro_generator)) + resource_type=NodeType.Analysis)) return parsed_models - def get_parsed_data_tests(self, root_project, all_projects, - macro_generator): + def get_parsed_data_tests(self, root_project, all_projects): parsed_tests = {} for name, project in all_projects.items(): @@ -490,7 +511,6 @@ def get_parsed_data_tests(self, root_project, all_projects, root_dir=project.get('project-root'), relative_dirs=project.get('test-paths', []), resource_type=NodeType.Test, - macro_generator=macro_generator, tags={'data'})) return parsed_tests @@ -509,16 +529,16 @@ def get_parsed_schema_tests(self, root_project, all_projects): return parsed_tests - def load_all_nodes(self, root_project, all_projects, macro_generator): + def load_all_macros(self, root_project, all_projects): + return self.get_parsed_macros(root_project, all_projects) + + def load_all_nodes(self, root_project, all_projects): all_nodes = {} - all_nodes.update(self.get_parsed_models(root_project, all_projects, - macro_generator)) - all_nodes.update(self.get_parsed_analyses(root_project, all_projects, - macro_generator)) + all_nodes.update(self.get_parsed_models(root_project, all_projects)) + all_nodes.update(self.get_parsed_analyses(root_project, all_projects)) all_nodes.update( - self.get_parsed_data_tests(root_project, all_projects, - macro_generator)) + self.get_parsed_data_tests(root_project, all_projects)) all_nodes.update( self.get_parsed_schema_tests(root_project, all_projects)) all_nodes.update( @@ -533,26 +553,25 @@ def compile(self): root_project = self.project.cfg all_projects = self.get_all_projects() - all_macros = self.get_macros(this_project=self.project) - - for project in dbt.utils.dependency_projects(self.project): - all_macros.extend( - self.get_macros(this_project=self.project, - own_project=project)) - - macro_generator = self.generate_macros(all_macros) + all_macros = self.load_all_macros(root_project, all_projects) + all_nodes = self.load_all_nodes(root_project, all_projects) - all_nodes = self.load_all_nodes(root_project, all_projects, - macro_generator) + flat_graph = { + 'nodes': all_nodes, + 'macros': all_macros + } - compiled_nodes, written_nodes = self.compile_nodes(linker, all_nodes, - macro_generator) + compiled_graph, written_nodes = self.compile_graph(linker, flat_graph) self.write_graph_file(linker) stats = {} - for node_name, node in compiled_nodes.items(): + for node_name, node in compiled_graph.get('nodes').items(): + stats[node.get('resource_type')] = stats.get( + node.get('resource_type'), 0) + 1 + + for node_name, node in compiled_graph.get('macros').items(): stats[node.get('resource_type')] = stats.get( node.get('resource_type'), 0) + 1 diff --git a/dbt/contracts/graph/compiled.py b/dbt/contracts/graph/compiled.py index 05ce178c1eb..ea65184f064 100644 --- a/dbt/contracts/graph/compiled.py +++ b/dbt/contracts/graph/compiled.py @@ -1,5 +1,4 @@ -from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \ - Length +from voluptuous import Schema, Required, All, Any, Length from collections import OrderedDict @@ -8,10 +7,11 @@ from dbt.logger import GLOBAL_LOGGER as logger from dbt.contracts.common import validate_with -from dbt.contracts.graph.parsed import parsed_graph_item_contract +from dbt.contracts.graph.parsed import parsed_node_contract, \ + parsed_macro_contract -compiled_graph_item_contract = parsed_graph_item_contract.extend({ +compiled_node_contract = parsed_node_contract.extend({ # compiled fields Required('compiled'): bool, Required('compiled_sql'): Any(basestring, None), @@ -24,16 +24,25 @@ Required('injected_sql'): Any(basestring, None), }) +compiled_nodes_contract = Schema({ + str: compiled_node_contract, +}) -def validate_one(compiled_graph_item): - validate_with(compiled_graph_item_contract, compiled_graph_item) +compiled_macro_contract = parsed_macro_contract +compiled_macros_contract = Schema({ + str: compiled_macro_contract, +}) -def validate(compiled_graph): - for k, v in compiled_graph.items(): - validate_with(compiled_graph_item_contract, v) +compiled_graph_contract = Schema({ + Required('nodes'): compiled_nodes_contract, + Required('macros'): compiled_macros_contract, +}) - if v.get('unique_id') != k: - error_msg = 'unique_id must match key name in compiled graph!' - logger.info(error_msg) - raise ValidationException(error_msg) + +def validate_node(compiled_node): + validate_with(compiled_node_contract, compiled_node) + + +def validate(compiled_graph): + validate_with(compiled_graph_contract, compiled_graph) diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 55d263eccc3..969ec818705 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -1,12 +1,15 @@ -from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \ - Length +from voluptuous import Schema, Required, All, Any, Length, Optional + +import jinja2.runtime from dbt.compat import basestring -from dbt.exceptions import ValidationException -from dbt.logger import GLOBAL_LOGGER as logger +from dbt.model import NodeType from dbt.contracts.common import validate_with -from dbt.contracts.graph.unparsed import unparsed_graph_item_contract +from dbt.contracts.graph.unparsed import unparsed_node_contract, \ + unparsed_base_contract + +from dbt.logger import GLOBAL_LOGGER as logger # noqa config_contract = { @@ -25,40 +28,54 @@ Optional('dist'): basestring, } - -parsed_graph_item_contract = unparsed_graph_item_contract.extend({ +parsed_node_contract = unparsed_node_contract.extend({ # identifiers Required('unique_id'): All(basestring, Length(min=1, max=255)), Required('fqn'): All(list, [All(basestring)]), # parsed fields - Required('depends_on'): All(list, - [All(basestring, Length(min=1, max=255))]), + Required('depends_on'): { + Required('nodes'): [All(basestring, Length(min=1, max=255))], + Required('macros'): [All(basestring, Length(min=1, max=255))], + }, Required('empty'): bool, Required('config'): config_contract, Required('tags'): All(set), }) +parsed_nodes_contract = Schema({ + str: parsed_node_contract, +}) + +parsed_macro_contract = unparsed_base_contract.extend({ + # identifiers + Required('resource_type'): Any(NodeType.Macro), + Required('unique_id'): All(basestring, Length(min=1, max=255)), + Required('tags'): All(set), + + # contents + Required('parsed_macro'): jinja2.runtime.Macro + +}) + +parsed_macros_contract = Schema({ + str: parsed_macro_contract, +}) + + +parsed_graph_contract = Schema({ + Required('nodes'): parsed_nodes_contract, + Required('macros'): parsed_macros_contract, +}) + -def validate_one(parsed_graph_item): - validate_with(parsed_graph_item_contract, parsed_graph_item) +def validate_nodes(parsed_nodes): + validate_with(parsed_nodes_contract, parsed_nodes) - materialization = parsed_graph_item.get('config', {}) \ - .get('materialized') - if materialization == 'incremental' and \ - parsed_graph_item.get('config', {}).get('sql_where') is None: - raise ValidationException( - 'missing `sql_where` for an incremental model') +def validate_macros(parsed_macros): + validate_with(parsed_macros_contract, parsed_macros) def validate(parsed_graph): - for k, v in parsed_graph.items(): - validate_one(v) - - if v.get('unique_id') != k: - error_msg = ('unique_id must match key name in parsed graph!' - 'key: {}, model: {}' - .format(k, v)) - logger.info(error_msg) - raise ValidationException(error_msg) + validate_with(parsed_graph_contract, parsed_graph) diff --git a/dbt/contracts/graph/unparsed.py b/dbt/contracts/graph/unparsed.py index 0ff7bb8cae9..bf0ff995094 100644 --- a/dbt/contracts/graph/unparsed.py +++ b/dbt/contracts/graph/unparsed.py @@ -1,19 +1,14 @@ -from voluptuous import Schema, Required, All, Any, Extra, Range, Optional, \ - Length +from voluptuous import Schema, Required, All, Any, Length from dbt.compat import basestring from dbt.contracts.common import validate_with -from dbt.logger import GLOBAL_LOGGER as logger from dbt.model import NodeType -unparsed_graph_item_contract = Schema({ +unparsed_base_contract = Schema({ # identifiers Required('name'): All(basestring, Length(min=1, max=127)), Required('package_name'): basestring, - Required('resource_type'): Any(NodeType.Model, - NodeType.Test, - NodeType.Analysis), # filesystem Required('root_path'): basestring, @@ -21,7 +16,14 @@ Required('raw_sql'): basestring, }) +unparsed_node_contract = unparsed_base_contract.extend({ + Required('resource_type'): Any(NodeType.Model, + NodeType.Test, + NodeType.Analysis) +}) + +unparsed_nodes_contract = Schema([unparsed_node_contract]) + -def validate(unparsed_graph): - for item in unparsed_graph: - validate_with(unparsed_graph_item_contract, item) +def validate_nodes(nodes): + validate_with(unparsed_nodes_contract, nodes) diff --git a/dbt/model.py b/dbt/model.py index e341e12ff22..553712cb611 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -20,6 +20,7 @@ class NodeType(object): Analysis = 'analysis' Test = 'test' Archive = 'archive' + Macro = 'macro' class SourceConfig(object): diff --git a/dbt/parser.py b/dbt/parser.py index 210e0551df3..38dd89885c6 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -6,6 +6,7 @@ import dbt.model import dbt.utils +import jinja2.runtime import dbt.clients.jinja import dbt.contracts.graph.parsed @@ -78,14 +79,6 @@ def get_macro_path(package_name, resource_name): return get_path('macros', package_name, resource_name) -def __ref(model): - - def ref(*args): - pass - - return ref - - def __config(model, cfg): def config(*args, **kwargs): @@ -114,8 +107,49 @@ def get_fqn(path, package_project_config, extra=[]): return fqn +def parse_macro_file(macro_file_path, + macro_file_contents, + root_path, + package_name, + tags=None): + + logger.debug("Parsing {}".format(macro_file_path)) + + to_return = {} + + if tags is None: + tags = set() + + template = dbt.clients.jinja.get_template(macro_file_contents, { + 'ref': lambda *args: '', + 'var': lambda *args: '', + 'target': property(lambda x: '', lambda x: x), + 'this': '' + }) + + for key, item in template.module.__dict__.items(): + if type(item) == jinja2.runtime.Macro: + unique_id = get_path(NodeType.Macro, + package_name, + key) + + to_return[unique_id] = { + 'name': key, + 'unique_id': unique_id, + 'tags': tags, + 'package_name': package_name, + 'resource_type': NodeType.Macro, + 'root_path': root_path, + 'path': macro_file_path, + 'raw_sql': macro_file_contents, + 'parsed_macro': item + } + + return to_return + + def parse_node(node, node_path, root_project_config, package_project_config, - macro_generator=None, tags=None, fqn_extra=None): + tags=None, fqn_extra=None): logger.debug("Parsing {}".format(node_path)) parsed_node = copy.deepcopy(node) @@ -126,7 +160,10 @@ def parse_node(node, node_path, root_project_config, package_project_config, fqn_extra = [] parsed_node.update({ - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [], + } }) fqn = get_fqn(node.get('path'), package_project_config, fqn_extra) @@ -136,26 +173,14 @@ def parse_node(node, node_path, root_project_config, package_project_config, context = {} - context['ref'] = __ref(parsed_node) + context['ref'] = lambda *args: '' context['config'] = __config(parsed_node, config) context['var'] = lambda *args: '' context['target'] = property(lambda x: '', lambda x: x) context['this'] = '' - if macro_generator is not None: - for macro_data in macro_generator(context): - macro = macro_data["macro"] - macro_name = macro_data["name"] - project = macro_data["project"] - - if context.get(project.get('name')) is None: - context[project.get('name')] = {} - - context.get(project.get('name'), {}) \ - .update({macro_name: macro}) - - if node.get('package_name') == project.get('name'): - context.update({macro_name: macro}) + def undefined_callback(name): + pass dbt.clients.jinja.get_rendered( node.get('raw_sql'), context, node, silent_on_undefined=True) @@ -172,15 +197,13 @@ def parse_node(node, node_path, root_project_config, package_project_config, return parsed_node -def parse_sql_nodes(nodes, root_project, projects, macro_generator=None, - tags=None): - +def parse_sql_nodes(nodes, root_project, projects, tags=None): if tags is None: tags = set() to_return = {} - dbt.contracts.graph.unparsed.validate(nodes) + dbt.contracts.graph.unparsed.validate_nodes(nodes) for node in nodes: package_name = node.get('package_name') @@ -194,17 +217,15 @@ def parse_sql_nodes(nodes, root_project, projects, macro_generator=None, node_path, root_project, projects.get(package_name), - macro_generator, tags=tags) - dbt.contracts.graph.parsed.validate(to_return) + dbt.contracts.graph.parsed.validate_nodes(to_return) return to_return def load_and_parse_sql(package_name, root_project, all_projects, root_dir, - relative_dirs, resource_type, macro_generator, - tags=None): + relative_dirs, resource_type, tags=None): extension = "[!.#~]*.sql" if tags is None: @@ -242,8 +263,40 @@ def load_and_parse_sql(package_name, root_project, all_projects, root_dir, 'raw_sql': file_contents }) - return parse_sql_nodes(result, root_project, all_projects, macro_generator, - tags) + return parse_sql_nodes(result, root_project, all_projects, tags) + + +def load_and_parse_macros(package_name, root_project, all_projects, root_dir, + relative_dirs, resource_type, tags=None): + extension = "[!.#~]*.sql" + + if tags is None: + tags = set() + + if dbt.flags.STRICT_MODE: + dbt.contracts.project.validate_list(all_projects) + + file_matches = dbt.clients.system.find_matching( + root_dir, + relative_dirs, + extension) + + result = {} + + for file_match in file_matches: + file_contents = dbt.clients.system.load_file_contents( + file_match.get('absolute_path')) + + result.update( + parse_macro_file( + file_match.get('relative_path'), + file_contents, + root_dir, + package_name)) + + dbt.contracts.graph.parsed.validate_macros(result) + + return result def parse_schema_tests(tests, root_project, projects): diff --git a/dbt/utils.py b/dbt/utils.py index c3a3e7629f0..db5cb52db46 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -109,10 +109,10 @@ def model_cte_name(model): return '__dbt__CTE__{}'.format(model.get('name')) -def find_model_by_name(all_models, target_model_name, +def find_model_by_name(flat_graph, target_model_name, target_model_package): - for name, model in all_models.items(): + for name, model in flat_graph.get('nodes').items(): resource_type, package_name, model_name = name.split('.') if (resource_type == 'model' and diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index f56eab1a8ea..46825cfd480 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -1,4 +1,3 @@ -from mock import MagicMock import unittest import os @@ -45,57 +44,67 @@ def test__prepend_ctes__already_has_cte(self): ephemeral_config = self.model_config.copy() ephemeral_config['materialized'] = 'ephemeral' - compiled_models = { - 'model.root.view': { - 'name': 'view', - 'resource_type': 'model', - 'unique_id': 'model.root.view', - 'fqn': ['root_project', 'view'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [ - 'model.root.ephemeral' - ], - 'config': self.model_config, - 'tags': set(), - 'path': 'view.sql', - 'raw_sql': 'select * from {{ref("ephemeral")}}', - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict([ - ('model.root.ephemeral', None) - ]), - 'injected_sql': '', - 'compiled_sql': ('with cte as (select * from something_else) ' - 'select * from __dbt__CTE__ephemeral') - }, - 'model.root.ephemeral': { - 'name': 'ephemeral', - 'resource_type': 'model', - 'unique_id': 'model.root.ephemeral', - 'fqn': ['root_project', 'ephemeral'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [], - 'config': ephemeral_config, - 'tags': set(), - 'path': 'ephemeral.sql', - 'raw_sql': 'select * from source_table', - 'compiled': True, - 'compiled_sql': 'select * from source_table', - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict(), - 'injected_sql': '' + input_graph = { + 'macros': {}, + 'nodes': { + 'model.root.view': { + 'name': 'view', + 'resource_type': 'model', + 'unique_id': 'model.root.view', + 'fqn': ['root_project', 'view'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [ + 'model.root.ephemeral' + ], + 'macros': [] + }, + 'config': self.model_config, + 'tags': set(), + 'path': 'view.sql', + 'raw_sql': 'select * from {{ref("ephemeral")}}', + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict([ + ('model.root.ephemeral', None) + ]), + 'injected_sql': '', + 'compiled_sql': ( + 'with cte as (select * from something_else) ' + 'select * from __dbt__CTE__ephemeral') + }, + 'model.root.ephemeral': { + 'name': 'ephemeral', + 'resource_type': 'model', + 'unique_id': 'model.root.ephemeral', + 'fqn': ['root_project', 'ephemeral'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [], + 'macros': [] + }, + 'config': ephemeral_config, + 'tags': set(), + 'path': 'ephemeral.sql', + 'raw_sql': 'select * from source_table', + 'compiled': True, + 'compiled_sql': 'select * from source_table', + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict(), + 'injected_sql': '' + } } } - result, all_models = dbt.compilation.prepend_ctes( - compiled_models['model.root.view'], - compiled_models) + result, output_graph = dbt.compilation.prepend_ctes( + input_graph['nodes']['model.root.view'], + input_graph) - self.assertEqual(result, all_models.get('model.root.view')) + self.assertEqual(result, output_graph['nodes']['model.root.view']) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), @@ -105,127 +114,158 @@ def test__prepend_ctes__already_has_cte(self): 'select * from __dbt__CTE__ephemeral')) self.assertEqual( - all_models.get('model.root.ephemeral').get('extra_ctes_injected'), + (input_graph.get('nodes', {}) + .get('model.root.ephemeral', {}) + .get('extra_ctes_injected')), True) def test__prepend_ctes__no_ctes(self): - compiled_models = { - 'model.root.view': { - 'name': 'view', - 'resource_type': 'model', - 'unique_id': 'model.root.view', - 'fqn': ['root_project', 'view'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [], - 'config': self.model_config, - 'tags': set(), - 'path': 'view.sql', - 'raw_sql': ('with cte as (select * from something_else) ' - 'select * from source_table'), - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict(), - 'injected_sql': '', - 'compiled_sql': ('with cte as (select * from something_else) ' - 'select * from source_table') - }, - 'model.root.view_no_cte': { - 'name': 'view_no_cte', - 'resource_type': 'model', - 'unique_id': 'model.root.view_no_cte', - 'fqn': ['root_project', 'view_no_cte'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [], - 'config': self.model_config, - 'tags': set(), - 'path': 'view.sql', - 'raw_sql': 'select * from source_table', - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict(), - 'injected_sql': '', - 'compiled_sql': ('select * from source_table') + input_graph = { + 'macros': {}, + 'nodes': { + 'model.root.view': { + 'name': 'view', + 'resource_type': 'model', + 'unique_id': 'model.root.view', + 'fqn': ['root_project', 'view'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [], + 'macros': [] + }, + 'config': self.model_config, + 'tags': set(), + 'path': 'view.sql', + 'raw_sql': ('with cte as (select * from something_else) ' + 'select * from source_table'), + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict(), + 'injected_sql': '', + 'compiled_sql': ('with cte as (select * from something_else) ' + 'select * from source_table') + }, + 'model.root.view_no_cte': { + 'name': 'view_no_cte', + 'resource_type': 'model', + 'unique_id': 'model.root.view_no_cte', + 'fqn': ['root_project', 'view_no_cte'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [], + 'macros': [] + }, + 'config': self.model_config, + 'tags': set(), + 'path': 'view.sql', + 'raw_sql': 'select * from source_table', + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict(), + 'injected_sql': '', + 'compiled_sql': ('select * from source_table') + } } } - result, all_models = dbt.compilation.prepend_ctes( - compiled_models.get('model.root.view'), - compiled_models) + result, output_graph = dbt.compilation.prepend_ctes( + input_graph.get('nodes').get('model.root.view'), + input_graph) - self.assertEqual(result, all_models.get('model.root.view')) + self.assertEqual( + result, + output_graph.get('nodes').get('model.root.view')) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), - compiled_models.get('model.root.view').get('compiled_sql')) + (output_graph.get('nodes') + .get('model.root.view') + .get('compiled_sql'))) - result, all_models = dbt.compilation.prepend_ctes( - compiled_models.get('model.root.view_no_cte'), - compiled_models) + result, output_graph = dbt.compilation.prepend_ctes( + input_graph.get('nodes').get('model.root.view_no_cte'), + input_graph) - self.assertEqual(result, all_models.get('model.root.view_no_cte')) + self.assertEqual( + result, + output_graph.get('nodes').get('model.root.view_no_cte')) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), - compiled_models.get('model.root.view_no_cte').get('compiled_sql')) + (output_graph.get('nodes') + .get('model.root.view_no_cte') + .get('compiled_sql'))) def test__prepend_ctes(self): ephemeral_config = self.model_config.copy() ephemeral_config['materialized'] = 'ephemeral' - compiled_models = { - 'model.root.view': { - 'name': 'view', - 'resource_type': 'model', - 'unique_id': 'model.root.view', - 'fqn': ['root_project', 'view'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [ - 'model.root.ephemeral' - ], - 'config': self.model_config, - 'tags': set(), - 'path': 'view.sql', - 'raw_sql': 'select * from {{ref("ephemeral")}}', - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict([ - ('model.root.ephemeral', None) - ]), - 'injected_sql': '', - 'compiled_sql': 'select * from __dbt__CTE__ephemeral' - }, - 'model.root.ephemeral': { - 'name': 'ephemeral', - 'resource_type': 'model', - 'unique_id': 'model.root.ephemeral', - 'fqn': ['root_project', 'ephemeral'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [], - 'config': ephemeral_config, - 'tags': set(), - 'path': 'ephemeral.sql', - 'raw_sql': 'select * from source_table', - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict(), - 'injected_sql': '', - 'compiled_sql': 'select * from source_table' + input_graph = { + 'macros': {}, + 'nodes': { + 'model.root.view': { + 'name': 'view', + 'resource_type': 'model', + 'unique_id': 'model.root.view', + 'fqn': ['root_project', 'view'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [ + 'model.root.ephemeral' + ], + 'macros': [] + }, + 'config': self.model_config, + 'tags': set(), + 'path': 'view.sql', + 'raw_sql': 'select * from {{ref("ephemeral")}}', + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict([ + ('model.root.ephemeral', None) + ]), + 'injected_sql': '', + 'compiled_sql': 'select * from __dbt__CTE__ephemeral' + }, + 'model.root.ephemeral': { + 'name': 'ephemeral', + 'resource_type': 'model', + 'unique_id': 'model.root.ephemeral', + 'fqn': ['root_project', 'ephemeral'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [], + 'macros': [] + }, + 'config': ephemeral_config, + 'tags': set(), + 'path': 'ephemeral.sql', + 'raw_sql': 'select * from source_table', + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict(), + 'injected_sql': '', + 'compiled_sql': 'select * from source_table' + } } } - result, all_models = dbt.compilation.prepend_ctes( - compiled_models['model.root.view'], - compiled_models) + result, output_graph = dbt.compilation.prepend_ctes( + input_graph.get('nodes').get('model.root.view'), + input_graph) + + self.assertEqual(result, + (output_graph.get('nodes', {}) + .get('model.root.view'))) - self.assertEqual(result, all_models.get('model.root.view')) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), @@ -235,86 +275,98 @@ def test__prepend_ctes(self): 'select * from __dbt__CTE__ephemeral')) self.assertEqual( - all_models.get('model.root.ephemeral').get('extra_ctes_injected'), + (output_graph.get('nodes', {}) + .get('model.root.ephemeral', {}) + .get('extra_ctes_injected')), True) - def test__prepend_ctes__multiple_levels(self): ephemeral_config = self.model_config.copy() ephemeral_config['materialized'] = 'ephemeral' - compiled_models = { - 'model.root.view': { - 'name': 'view', - 'resource_type': 'model', - 'unique_id': 'model.root.view', - 'fqn': ['root_project', 'view'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [ - 'model.root.ephemeral' - ], - 'config': self.model_config, - 'tags': set(), - 'path': 'view.sql', - 'raw_sql': 'select * from {{ref("ephemeral")}}', - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict([ - ('model.root.ephemeral', None) - ]), - 'injected_sql': '', - 'compiled_sql': 'select * from __dbt__CTE__ephemeral' - }, - 'model.root.ephemeral': { - 'name': 'ephemeral', - 'resource_type': 'model', - 'unique_id': 'model.root.ephemeral', - 'fqn': ['root_project', 'ephemeral'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [], - 'config': ephemeral_config, - 'tags': set(), - 'path': 'ephemeral.sql', - 'raw_sql': 'select * from {{ref("ephemeral_level_two")}}', - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict([ - ('model.root.ephemeral_level_two', None) - ]), - 'injected_sql': '', - 'compiled_sql': 'select * from __dbt__CTE__ephemeral_level_two' - }, - 'model.root.ephemeral_level_two': { - 'name': 'ephemeral_level_two', - 'resource_type': 'model', - 'unique_id': 'model.root.ephemeral_level_two', - 'fqn': ['root_project', 'ephemeral_level_two'], - 'empty': False, - 'package_name': 'root', - 'root_path': '/usr/src/app', - 'depends_on': [], - 'config': ephemeral_config, - 'tags': set(), - 'path': 'ephemeral_level_two.sql', - 'raw_sql': 'select * from source_table', - 'compiled': True, - 'extra_ctes_injected': False, - 'extra_ctes': OrderedDict(), - 'injected_sql': '', - 'compiled_sql': 'select * from source_table' + input_graph = { + 'macros': {}, + 'nodes': { + 'model.root.view': { + 'name': 'view', + 'resource_type': 'model', + 'unique_id': 'model.root.view', + 'fqn': ['root_project', 'view'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [ + 'model.root.ephemeral' + ], + 'macros': [] + }, + 'config': self.model_config, + 'tags': set(), + 'path': 'view.sql', + 'raw_sql': 'select * from {{ref("ephemeral")}}', + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict([ + ('model.root.ephemeral', None) + ]), + 'injected_sql': '', + 'compiled_sql': 'select * from __dbt__CTE__ephemeral' + }, + 'model.root.ephemeral': { + 'name': 'ephemeral', + 'resource_type': 'model', + 'unique_id': 'model.root.ephemeral', + 'fqn': ['root_project', 'ephemeral'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [], + 'macros': [] + }, + 'config': ephemeral_config, + 'tags': set(), + 'path': 'ephemeral.sql', + 'raw_sql': 'select * from {{ref("ephemeral_level_two")}}', + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict([ + ('model.root.ephemeral_level_two', None) + ]), + 'injected_sql': '', + 'compiled_sql': 'select * from __dbt__CTE__ephemeral_level_two' # noqa + }, + 'model.root.ephemeral_level_two': { + 'name': 'ephemeral_level_two', + 'resource_type': 'model', + 'unique_id': 'model.root.ephemeral_level_two', + 'fqn': ['root_project', 'ephemeral_level_two'], + 'empty': False, + 'package_name': 'root', + 'root_path': '/usr/src/app', + 'depends_on': { + 'nodes': [], + 'macros': [] + }, + 'config': ephemeral_config, + 'tags': set(), + 'path': 'ephemeral_level_two.sql', + 'raw_sql': 'select * from source_table', + 'compiled': True, + 'extra_ctes_injected': False, + 'extra_ctes': OrderedDict(), + 'injected_sql': '', + 'compiled_sql': 'select * from source_table' + } } - } - result, all_models = dbt.compilation.prepend_ctes( - compiled_models['model.root.view'], - compiled_models) + result, output_graph = dbt.compilation.prepend_ctes( + input_graph['nodes']['model.root.view'], + input_graph) - self.assertEqual(result, all_models.get('model.root.view')) + self.assertEqual(result, input_graph['nodes']['model.root.view']) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), @@ -326,8 +378,12 @@ def test__prepend_ctes__multiple_levels(self): 'select * from __dbt__CTE__ephemeral')) self.assertEqual( - all_models.get('model.root.ephemeral').get('extra_ctes_injected'), + (output_graph.get('nodes') + .get('model.root.ephemeral') + .get('extra_ctes_injected')), True) self.assertEqual( - all_models.get('model.root.ephemeral_level_two').get('extra_ctes_injected'), + (output_graph.get('nodes') + .get('model.root.ephemeral_level_two') + .get('extra_ctes_injected')), True) diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index f0142420b3e..3d864ab37b5 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -15,7 +15,7 @@ import networkx as nx from test.integration.base import FakeArgs -# from dbt.logger import GLOBAL_LOGGER as logger +from dbt.logger import GLOBAL_LOGGER as logger # noqa class GraphTest(unittest.TestCase): diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 63c86360c96..6507b49a299 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -1,4 +1,3 @@ -from mock import MagicMock import unittest import os @@ -69,7 +68,10 @@ def test__single_model(self): 'empty': False, 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'model_one.sql', @@ -126,7 +128,10 @@ def test__single_model__nested_configuration(self): 'empty': False, 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': ephemeral_config, 'tags': set(), 'path': get_os_path('nested/path/model_one.sql'), @@ -159,7 +164,10 @@ def test__empty_model(self): 'fqn': ['root', 'model_one'], 'empty': True, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [], + }, 'config': self.model_config, 'tags': set(), 'path': 'model_one.sql', @@ -201,7 +209,10 @@ def test__simple_dependency(self): 'fqn': ['root', 'base'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'base.sql', @@ -216,7 +227,10 @@ def test__simple_dependency(self): 'fqn': ['root', 'events_tx'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'events_tx.sql', @@ -283,7 +297,10 @@ def test__multiple_dependencies(self): 'fqn': ['root', 'events'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'events.sql', @@ -298,7 +315,10 @@ def test__multiple_dependencies(self): 'fqn': ['root', 'sessions'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'sessions.sql', @@ -313,7 +333,10 @@ def test__multiple_dependencies(self): 'fqn': ['root', 'events_tx'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'events_tx.sql', @@ -328,7 +351,10 @@ def test__multiple_dependencies(self): 'fqn': ['root', 'sessions_tx'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'sessions_tx.sql', @@ -343,7 +369,10 @@ def test__multiple_dependencies(self): 'fqn': ['root', 'multi'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'multi.sql', @@ -391,8 +420,10 @@ def test__multiple_dependencies__packages(self): 'package_name': 'root', 'path': 'multi.sql', 'root_path': get_os_path('/usr/src/app'), - 'raw_sql': ("with s as (select * from {{ref('snowplow', 'sessions_tx')}}), " - "e as (select * from {{ref('snowplow', 'events_tx')}}) " + 'raw_sql': ("with s as " + "(select * from {{ref('snowplow', 'sessions_tx')}}), " + "e as " + "(select * from {{ref('snowplow', 'events_tx')}}) " "select * from e left join s on s.id = e.sid"), }] @@ -410,7 +441,10 @@ def test__multiple_dependencies__packages(self): 'fqn': ['snowplow', 'events'], 'empty': False, 'package_name': 'snowplow', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'events.sql', @@ -425,7 +459,10 @@ def test__multiple_dependencies__packages(self): 'fqn': ['snowplow', 'sessions'], 'empty': False, 'package_name': 'snowplow', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'sessions.sql', @@ -440,7 +477,10 @@ def test__multiple_dependencies__packages(self): 'fqn': ['snowplow', 'events_tx'], 'empty': False, 'package_name': 'snowplow', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'events_tx.sql', @@ -455,7 +495,10 @@ def test__multiple_dependencies__packages(self): 'fqn': ['snowplow', 'sessions_tx'], 'empty': False, 'package_name': 'snowplow', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'sessions_tx.sql', @@ -470,7 +513,10 @@ def test__multiple_dependencies__packages(self): 'fqn': ['root', 'multi'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'tags': set(), 'path': 'multi.sql', @@ -510,7 +556,10 @@ def test__in_model_config(self): 'fqn': ['root', 'model_one'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [], + }, 'config': self.model_config, 'tags': set(), 'root_path': get_os_path('/usr/src/app'), @@ -589,7 +638,10 @@ def test__root_project_config(self): 'fqn': ['root', 'table'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': 'table.sql', 'config': self.model_config, 'tags': set(), @@ -604,7 +656,10 @@ def test__root_project_config(self): 'fqn': ['root', 'ephemeral'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': 'ephemeral.sql', 'config': ephemeral_config, 'tags': set(), @@ -619,7 +674,10 @@ def test__root_project_config(self): 'fqn': ['root', 'view'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': 'view.sql', 'root_path': get_os_path('/usr/src/app'), 'config': view_config, @@ -747,7 +805,10 @@ def test__other_project_config(self): 'fqn': ['root', 'table'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': 'table.sql', 'root_path': get_os_path('/usr/src/app'), 'config': self.model_config, @@ -762,7 +823,10 @@ def test__other_project_config(self): 'fqn': ['root', 'ephemeral'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': 'ephemeral.sql', 'root_path': get_os_path('/usr/src/app'), 'config': ephemeral_config, @@ -777,7 +841,10 @@ def test__other_project_config(self): 'fqn': ['root', 'view'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': 'view.sql', 'root_path': get_os_path('/usr/src/app'), 'config': view_config, @@ -792,7 +859,10 @@ def test__other_project_config(self): 'fqn': ['snowplow', 'disabled'], 'empty': False, 'package_name': 'snowplow', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': 'disabled.sql', 'root_path': get_os_path('/usr/src/app'), 'config': disabled_config, @@ -807,7 +877,10 @@ def test__other_project_config(self): 'fqn': ['snowplow', 'views', 'package'], 'empty': False, 'package_name': 'snowplow', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'path': get_os_path('views/package.sql'), 'root_path': get_os_path('/usr/src/app'), 'config': sort_config, @@ -850,11 +923,11 @@ def test__simple_schema_test(self): values_csv="'a','b'") relationships_sql = dbt.parser.QUERY_VALIDATE_REFERENTIAL_INTEGRITY \ - .format( - parent_field='id', - parent_ref="{{ref('model_two')}}", - child_field='id', - child_ref="{{ref('model_one')}}") + .format( + parent_field='id', + parent_ref="{{ref('model_two')}}", + child_field='id', + child_ref="{{ref('model_one')}}") self.assertEquals( dbt.parser.parse_schema_tests( @@ -871,9 +944,13 @@ def test__simple_schema_test(self): 'empty': False, 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, - 'path': get_os_path('schema_test/not_null_model_one_id.sql'), + 'path': get_os_path( + 'schema_test/not_null_model_one_id.sql'), 'tags': set(['schema']), 'raw_sql': not_null_sql, }, @@ -885,7 +962,10 @@ def test__simple_schema_test(self): 'empty': False, 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'path': get_os_path('schema_test/unique_model_one_id.sql'), 'tags': set(['schema']), @@ -895,27 +975,36 @@ def test__simple_schema_test(self): 'name': 'accepted_values_model_one_id', 'resource_type': 'test', 'unique_id': 'test.root.accepted_values_model_one_id', - 'fqn': ['root', 'schema_test', 'accepted_values_model_one_id'], + 'fqn': ['root', 'schema_test', + 'accepted_values_model_one_id'], 'empty': False, 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, - 'path': get_os_path('schema_test/accepted_values_model_one_id.sql'), + 'path': get_os_path( + 'schema_test/accepted_values_model_one_id.sql'), 'tags': set(['schema']), 'raw_sql': accepted_values_sql, }, 'test.root.relationships_model_one_id_to_model_two_id': { 'name': 'relationships_model_one_id_to_model_two_id', 'resource_type': 'test', - 'unique_id': 'test.root.relationships_model_one_id_to_model_two_id', - 'fqn': ['root', 'schema_test', 'relationships_model_one_id_to_model_two_id'], + 'unique_id': 'test.root.relationships_model_one_id_to_model_two_id', # noqa + 'fqn': ['root', 'schema_test', + 'relationships_model_one_id_to_model_two_id'], 'empty': False, 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, - 'path': get_os_path('schema_test/relationships_model_one_id_to_model_two_id.sql'), + 'path': get_os_path('schema_test/relationships_model_one_id_to_model_two_id.sql'), # noqa 'tags': set(['schema']), 'raw_sql': relationships_sql, } @@ -924,7 +1013,6 @@ def test__simple_schema_test(self): } ) - def test__simple_data_test(self): tests = [{ 'name': 'no_events', @@ -949,7 +1037,10 @@ def test__simple_data_test(self): 'fqn': ['root', 'no_events'], 'empty': False, 'package_name': 'root', - 'depends_on': [], + 'depends_on': { + 'nodes': [], + 'macros': [] + }, 'config': self.model_config, 'path': 'no_events.sql', 'root_path': get_os_path('/usr/src/app'), @@ -959,3 +1050,41 @@ def test__simple_data_test(self): } } ) + + def test__macro_parsing(self): + models = [{ + 'name': 'model_one', + 'resource_type': 'model', + 'package_name': 'root', + 'root_path': get_os_path('/usr/src/app'), + 'path': 'model_one.sql', + 'raw_sql': ("select * from {{ macro_one('some_value') }}"), + }] + + self.assertEquals( + dbt.parser.parse_sql_nodes( + models, + self.root_project_config, + {'root': self.root_project_config, + 'snowplow': self.snowplow_project_config}), + { + 'model.root.model_one': { + 'name': 'model_one', + 'resource_type': 'model', + 'unique_id': 'model.root.model_one', + 'fqn': ['root', 'model_one'], + 'empty': False, + 'package_name': 'root', + 'root_path': get_os_path('/usr/src/app'), + 'depends_on': { + 'nodes': [], + 'macros': [] + }, + 'config': self.model_config, + 'tags': set(), + 'path': 'model_one.sql', + 'raw_sql': self.find_input_by_name( + models, 'model_one').get('raw_sql') + } + } + ) From 5b45a333cf17a449a1593f7c5739a53ad5facb68 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 15 Mar 2017 11:44:13 -0400 Subject: [PATCH 02/10] tests passing --- dbt/compilation.py | 54 ++++++++++++++--------------- dbt/parser.py | 21 +++++++++--- test/unit/test_graph.py | 5 +-- test/unit/test_parser.py | 74 ++++++++++++++++++++++++++++++++++++++-- 4 files changed, 115 insertions(+), 39 deletions(-) diff --git a/dbt/compilation.py b/dbt/compilation.py index d298995d2bb..7e5adbe3078 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -159,12 +159,6 @@ def initialize(self): if not os.path.exists(self.project['modules-path']): os.makedirs(self.project['modules-path']) - def get_macros(self, this_project, own_project=None): - if own_project is None: - own_project = this_project - paths = own_project.get('macro-paths', []) - return Source(this_project, own_project=own_project).get_macros(paths) - def __write(self, build_filepath, payload): target_path = os.path.join(self.project['target-path'], build_filepath) @@ -228,12 +222,12 @@ def wrapped_do_ref(*args): return wrapped_do_ref - def get_compiler_context(self, linker, model, models): + def get_compiler_context(self, linker, model, flat_graph): context = self.project.context() adapter = get_adapter(self.project.run_environment()) # built-ins - context['ref'] = self.__ref(context, model, models) + context['ref'] = self.__ref(context, model, flat_graph) context['config'] = self.__model_config(model, linker) context['this'] = This( context['env']['schema'], @@ -249,21 +243,32 @@ def get_compiler_context(self, linker, model, models): context['invocation_id'] = '{{ invocation_id }}' context['sql_now'] = adapter.date_function - macro_generator = None - if macro_generator is not None: - for macro_data in macro_generator(context): - macro = macro_data["macro"] - macro_name = macro_data["name"] - project = macro_data["project"] + for unique_id, macro in flat_graph.get('macros').items(): + name = macro.get('name') + package_name = macro.get('package_name') + + if context.get(package_name, {}).get(name) is not None: + # we've already re-parsed this macro and added it to + # the context. + continue - if context.get(project.get('name')) is None: - context[project.get('name')] = {} + reparsed = dbt.parser.parse_macro_file( + macro_file_path=macro.get('path'), + macro_file_contents=macro.get('raw_sql'), + root_path=macro.get('root_path'), + package_name=package_name) - context.get(project.get('name'), {}) \ - .update({macro_name: macro}) + for unique_id, macro in reparsed.items(): + macro_map = {macro.get('name'): macro.get('parsed_macro')} - if model.get('package_name') == project.get('name'): - context.update({macro_name: macro}) + if context.get(package_name) is None: + context[package_name] = {} + + context.get(package_name, {}) \ + .update(macro_map) + + if package_name == model.get('package_name'): + context.update(macro_map) return context @@ -431,15 +436,6 @@ def compile_graph(self, linker, flat_graph): return wrapped_graph, written_nodes - def generate_macros(self, all_macros): - def do_gen(ctx): - macros = [] - for macro in all_macros: - new_macros = macro.get_macros(ctx) - macros.extend(new_macros) - return macros - return do_gen - def get_all_projects(self): root_project = self.project.cfg all_projects = {root_project.get('name'): root_project} diff --git a/dbt/parser.py b/dbt/parser.py index 38dd89885c6..bd415f700bc 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -148,8 +148,13 @@ def parse_macro_file(macro_file_path, return to_return +class FakeProject: + def __call__(*args, **kwargs): + return None + + def parse_node(node, node_path, root_project_config, package_project_config, - tags=None, fqn_extra=None): + all_projects, tags=None, fqn_extra=None): logger.debug("Parsing {}".format(node_path)) parsed_node = copy.deepcopy(node) @@ -179,6 +184,9 @@ def parse_node(node, node_path, root_project_config, package_project_config, context['target'] = property(lambda x: '', lambda x: x) context['this'] = '' + for name, project in all_projects.items(): + context[name] = FakeProject() + def undefined_callback(name): pass @@ -217,6 +225,7 @@ def parse_sql_nodes(nodes, root_project, projects, tags=None): node_path, root_project, projects.get(package_name), + projects, tags=tags) dbt.contracts.graph.parsed.validate_nodes(to_return) @@ -313,7 +322,8 @@ def parse_schema_tests(tests, root_project, projects): to_add = parse_schema_test( test, model_name, config, test_type, root_project, - projects.get(test.get('package_name'))) + projects.get(test.get('package_name')), + all_projects=projects) if to_add is not None: to_return[to_add.get('unique_id')] = to_add @@ -322,7 +332,8 @@ def parse_schema_tests(tests, root_project, projects): def parse_schema_test(test_base, model_name, test_config, test_type, - root_project_config, package_project_config): + root_project_config, package_project_config, + all_projects): if test_type == 'not_null': raw_sql = QUERY_VALIDATE_NOT_NULL.format( ref="{{ref('"+model_name+"')}}", field=test_config) @@ -385,6 +396,7 @@ def parse_schema_test(test_base, model_name, test_config, test_type, name), root_project_config, package_project_config, + all_projects, tags={'schema'}, fqn_extra=None) @@ -438,7 +450,8 @@ def parse_archives_from_projects(root_project, all_projects): archive, node_path, root_project, - all_projects.get(archive.get('package_name'))) + all_projects.get(archive.get('package_name')), + all_projects) return to_return diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index 3d864ab37b5..efe23318f8e 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -105,13 +105,10 @@ def get_project(self, extra_cfg=None): return project def get_compiler(self, project): - compiler = dbt.compilation.Compiler( + return dbt.compilation.Compiler( project, FakeArgs()) - compiler.get_macros = MagicMock(return_value=[]) - return compiler - def use_models(self, models): for k, v in models.items(): path = os.path.abspath('models/{}.sql'.format(k)) diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 6507b49a299..9d4561b8982 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -1,5 +1,6 @@ import unittest +import jinja2.runtime import os import dbt.flags @@ -1051,14 +1052,83 @@ def test__simple_data_test(self): } ) - def test__macro_parsing(self): + def test__simple_macro(self): + macro_file_contents = """ +{% macro simple(a, b) %} + {{a}} + {{b}} +{% endmacro %} +""" + + result = dbt.parser.parse_macro_file( + macro_file_path='simple_macro.sql', + macro_file_contents=macro_file_contents, + root_path=get_os_path('/usr/src/app'), + package_name='root') + + self.assertEquals( + type(result.get('macro.root.simple', {}).get('parsed_macro')), + jinja2.runtime.Macro) + + del result['macro.root.simple']['parsed_macro'] + + self.assertEquals( + result, + { + 'macro.root.simple': { + 'name': 'simple', + 'resource_type': 'macro', + 'unique_id': 'macro.root.simple', + 'package_name': 'root', + 'root_path': get_os_path('/usr/src/app'), + 'tags': set(), + 'path': 'simple_macro.sql', + 'raw_sql': macro_file_contents, + } + } + ) + + def test__simple_macro_used_in_model(self): + macro_file_contents = """ +{% macro simple(a, b) %} + {{a}} + {{b}} +{% endmacro %} +""" + + result = dbt.parser.parse_macro_file( + macro_file_path='simple_macro.sql', + macro_file_contents=macro_file_contents, + root_path=get_os_path('/usr/src/app'), + package_name='root') + + self.assertEquals( + type(result.get('macro.root.simple', {}).get('parsed_macro')), + jinja2.runtime.Macro) + + del result['macro.root.simple']['parsed_macro'] + + self.assertEquals( + result, + { + 'macro.root.simple': { + 'name': 'simple', + 'resource_type': 'macro', + 'unique_id': 'macro.root.simple', + 'package_name': 'root', + 'root_path': get_os_path('/usr/src/app'), + 'tags': set(), + 'path': 'simple_macro.sql', + 'raw_sql': macro_file_contents, + } + } + ) + models = [{ 'name': 'model_one', 'resource_type': 'model', 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), 'path': 'model_one.sql', - 'raw_sql': ("select * from {{ macro_one('some_value') }}"), + 'raw_sql': ("select *, {{root.simple(1, 2)}} from events"), }] self.assertEquals( From 37994aaf1644f6f9ce56666c9ed9f08f71db807d Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 15 Mar 2017 11:52:25 -0400 Subject: [PATCH 03/10] self-review cleanup --- dbt/compilation.py | 2 +- dbt/parser.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dbt/compilation.py b/dbt/compilation.py index dec1e9ddf10..d39672944e3 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -424,7 +424,7 @@ def compile_graph(self, linker, flat_graph): injected_node.get('unique_id'), injected_node) - for dependency in injected_node['depends_on']['nodes']: + for dependency in injected_node.get('depends_on', {}).get('nodes'): if compiled_graph.get('nodes').get(dependency): linker.dependency( injected_node.get('unique_id'), diff --git a/dbt/parser.py b/dbt/parser.py index 6867324372d..f25b7b5c6e0 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -187,9 +187,6 @@ def parse_node(node, node_path, root_project_config, package_project_config, for name, project in all_projects.items(): context[name] = FakeProject() - def undefined_callback(name): - pass - dbt.clients.jinja.get_rendered( node.get('raw_sql'), context, node, silent_on_undefined=True) From 2fac65be02385840f0445bc7a3d3f0f9617e0f88 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 15 Mar 2017 14:51:43 -0400 Subject: [PATCH 04/10] only select macros that are used --- dbt/clients/jinja.py | 64 +++++++++++++++++++++------------ dbt/compilation.py | 11 ++++-- dbt/contracts/graph/parsed.py | 2 +- dbt/contracts/graph/unparsed.py | 2 +- dbt/exceptions.py | 7 ++++ dbt/graph/selector.py | 6 ++-- dbt/linker.py | 4 +-- dbt/model.py | 17 +++------ dbt/parser.py | 31 +++++++--------- dbt/runner.py | 13 ++----- dbt/utils.py | 35 +++++++++++++----- test/unit/test_parser.py | 46 ++++++++++++++++++++++-- 12 files changed, 153 insertions(+), 85 deletions(-) diff --git a/dbt/clients/jinja.py b/dbt/clients/jinja.py index 1903a5d145a..3ad0153c872 100644 --- a/dbt/clients/jinja.py +++ b/dbt/clients/jinja.py @@ -4,36 +4,56 @@ import jinja2 import jinja2.sandbox +from dbt.utils import NodeType -class SilentUndefined(jinja2.Undefined): - """ - This class sets up the parser to just ignore undefined jinja2 calls. So, - for example, `env` is not defined here, but will not make the parser fail - with a fatal error. - """ - def _fail_with_undefined_error(self, *args, **kwargs): - return None - __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ - __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ - __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ - __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = __int__ = \ - __float__ = __complex__ = __pow__ = __rpow__ = \ - _fail_with_undefined_error +def create_macro_capture_env(node): + class ParserMacroCapture(jinja2.Undefined): + """ + This class sets up the parser to capture macros. + """ + def __init__(self, hint=None, obj=None, name=None, + exc=None): + super(jinja2.Undefined, self).__init__() -env = jinja2.sandbox.SandboxedEnvironment() + self.node = node + self.name = name + self.package_name = node.get('package_name') + + def __getattr__(self, name): + + # jinja uses these for safety, so we have to override them. + # see https://github.com/pallets/jinja/blob/master/jinja2/sandbox.py#L332-L339 # noqa + if name in ['unsafe_callable', 'alters_data']: + return False + + self.package_name = self.name + self.name = name + + return self -silent_on_undefined_env = jinja2.sandbox.SandboxedEnvironment( - undefined=SilentUndefined) + def __call__(self, *args, **kwargs): + path = '{}.{}.{}'.format(NodeType.Macro, + self.package_name, + self.name) + + if path not in self.node['depends_on']['macros']: + self.node['depends_on']['macros'].append(path) + + return jinja2.sandbox.SandboxedEnvironment( + undefined=ParserMacroCapture) + + +env = jinja2.sandbox.SandboxedEnvironment() -def get_template(string, ctx, node=None, silent_on_undefined=False): +def get_template(string, ctx, node=None, capture_macros=False): try: local_env = env - if silent_on_undefined: - local_env = silent_on_undefined_env + if capture_macros is True: + local_env = create_macro_capture_env(node) return local_env.from_string(dbt.compat.to_string(string), globals=ctx) @@ -42,9 +62,9 @@ def get_template(string, ctx, node=None, silent_on_undefined=False): dbt.exceptions.raise_compiler_error(node, str(e)) -def get_rendered(string, ctx, node=None, silent_on_undefined=False): +def get_rendered(string, ctx, node=None, capture_macros=False): try: - template = get_template(string, ctx, node, silent_on_undefined) + template = get_template(string, ctx, node, capture_macros) return template.render(ctx) except (jinja2.exceptions.TemplateSyntaxError, diff --git a/dbt/compilation.py b/dbt/compilation.py index d39672944e3..674ffde089f 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -5,9 +5,9 @@ import dbt.project import dbt.utils -from dbt.model import Model, NodeType +from dbt.model import Model from dbt.source import Source -from dbt.utils import This, Var, is_enabled, get_materialization +from dbt.utils import This, Var, is_enabled, get_materialization, NodeType from dbt.linker import Linker from dbt.runtime import RuntimeContext @@ -245,7 +245,12 @@ def get_compiler_context(self, linker, model, flat_graph): context['invocation_id'] = '{{ invocation_id }}' context['sql_now'] = adapter.date_function - for unique_id, macro in flat_graph.get('macros').items(): + for unique_id in model.get('depends_on', {}).get('macros'): + macro = flat_graph.get('macros', {}).get(unique_id) + + if macro is None: + dbt.exceptions.macro_not_found(model, unique_id) + name = macro.get('name') package_name = macro.get('package_name') diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 969ec818705..3b2111cfe69 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -3,7 +3,7 @@ import jinja2.runtime from dbt.compat import basestring -from dbt.model import NodeType +from dbt.utils import NodeType from dbt.contracts.common import validate_with from dbt.contracts.graph.unparsed import unparsed_node_contract, \ diff --git a/dbt/contracts/graph/unparsed.py b/dbt/contracts/graph/unparsed.py index bf0ff995094..25415891755 100644 --- a/dbt/contracts/graph/unparsed.py +++ b/dbt/contracts/graph/unparsed.py @@ -3,7 +3,7 @@ from dbt.compat import basestring from dbt.contracts.common import validate_with -from dbt.model import NodeType +from dbt.utils import NodeType unparsed_base_contract = Schema({ # identifiers diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 22bdf2ca5be..5c341b14b73 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -73,3 +73,10 @@ def dependency_not_found(model, target_model_name): model, "'{}' depends on '{}' which is not in the graph!" .format(model.get('unique_id'), target_model_name)) + + +def macro_not_found(model, target_macro_id): + raise_compiler_error( + model, + "'{}' references macro '{}' which is not in the graph!" + .format(model.get('unique_id'), target_macro_id)) diff --git a/dbt/graph/selector.py b/dbt/graph/selector.py index 1fe227925bb..ca12c7c156f 100644 --- a/dbt/graph/selector.py +++ b/dbt/graph/selector.py @@ -1,10 +1,8 @@ - # import dbt.utils.compiler_error import networkx as nx from dbt.logger import GLOBAL_LOGGER as logger -import dbt.model - +from dbt.utils import NodeType SELECTOR_PARENTS = '+' SELECTOR_CHILDREN = '+' @@ -130,7 +128,7 @@ def get_nodes_from_spec(project, graph, spec): # they'll be filtered out later. child_tests = [n for n in graph.successors(node) if graph.node.get(n).get('resource_type') == - dbt.model.NodeType.Test] + NodeType.Test] test_nodes.update(child_tests) return model_nodes | test_nodes diff --git a/dbt/linker.py b/dbt/linker.py index fa12e3fa672..0ff9640e99d 100644 --- a/dbt/linker.py +++ b/dbt/linker.py @@ -2,7 +2,7 @@ from collections import defaultdict import dbt.compilation -import dbt.model +from dbt.utils import NodeType def from_file(graph_file): @@ -65,7 +65,7 @@ def is_blocking_dependency(self, node_data): if 'dbt_run_type' not in node_data or 'materialized' not in node_data: return False - return node_data['dbt_run_type'] == dbt.model.NodeType.Model \ + return node_data['dbt_run_type'] == NodeType.Model \ and node_data['materialized'] != 'ephemeral' def as_dependency_list(self, limit_to=None): diff --git a/dbt/model.py b/dbt/model.py index 553712cb611..2a6be8bfa79 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -4,23 +4,14 @@ from dbt.adapters.factory import get_adapter from dbt.compat import basestring -import dbt.clients.jinja import dbt.flags -from dbt.templates import BaseCreateTemplate, ArchiveInsertTemplate -from dbt.utils import split_path +from dbt.templates import BaseCreateTemplate +from dbt.utils import split_path, NodeType import dbt.project -from dbt.utils import deep_merge, DBTConfigKeys, compiler_error, \ - compiler_warning - +from dbt.utils import deep_merge, DBTConfigKeys, compiler_error -class NodeType(object): - Base = 'base' - Model = 'model' - Analysis = 'analysis' - Test = 'test' - Archive = 'archive' - Macro = 'macro' +import dbt.clients.jinja class SourceConfig(object): diff --git a/dbt/parser.py b/dbt/parser.py index f25b7b5c6e0..c521d33062a 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -13,7 +13,7 @@ import dbt.contracts.graph.unparsed import dbt.contracts.project -from dbt.model import NodeType +from dbt.utils import NodeType from dbt.logger import GLOBAL_LOGGER as logger QUERY_VALIDATE_NOT_NULL = """ @@ -148,15 +148,10 @@ def parse_macro_file(macro_file_path, return to_return -class FakeProject: - def __call__(*args, **kwargs): - return None - - def parse_node(node, node_path, root_project_config, package_project_config, all_projects, tags=None, fqn_extra=None): logger.debug("Parsing {}".format(node_path)) - parsed_node = copy.deepcopy(node) + node = copy.deepcopy(node) if tags is None: tags = set() @@ -164,7 +159,7 @@ def parse_node(node, node_path, root_project_config, package_project_config, if fqn_extra is None: fqn_extra = [] - parsed_node.update({ + node.update({ 'depends_on': { 'nodes': [], 'macros': [], @@ -179,27 +174,25 @@ def parse_node(node, node_path, root_project_config, package_project_config, context = {} context['ref'] = lambda *args: '' - context['config'] = __config(parsed_node, config) + context['config'] = __config(node, config) context['var'] = lambda *args: '' context['target'] = property(lambda x: '', lambda x: x) context['this'] = '' - for name, project in all_projects.items(): - context[name] = FakeProject() - dbt.clients.jinja.get_rendered( - node.get('raw_sql'), context, node, silent_on_undefined=True) + node.get('raw_sql'), context, node, + capture_macros=True) config_dict = node.get('config', {}) config_dict.update(config.config) - parsed_node['unique_id'] = node_path - parsed_node['config'] = config_dict - parsed_node['empty'] = (len(node.get('raw_sql').strip()) == 0) - parsed_node['fqn'] = fqn - parsed_node['tags'] = tags + node['unique_id'] = node_path + node['config'] = config_dict + node['empty'] = (len(node.get('raw_sql').strip()) == 0) + node['fqn'] = fqn + node['tags'] = tags - return parsed_node + return node def parse_sql_nodes(nodes, root_project, projects, tags=None): diff --git a/dbt/runner.py b/dbt/runner.py index 1a042136907..203c33a8a4f 100644 --- a/dbt/runner.py +++ b/dbt/runner.py @@ -3,21 +3,14 @@ import hashlib import psycopg2 import os -import sys -import logging import time import itertools -import re -import yaml from datetime import datetime from dbt.adapters.factory import get_adapter from dbt.logger import GLOBAL_LOGGER as logger -from dbt.source import Source -from dbt.utils import find_model_by_fqn, find_model_by_name, \ - dependency_projects, get_materialization -from dbt.model import NodeType +from dbt.utils import get_materialization, NodeType import dbt.clients.jinja import dbt.compilation @@ -162,13 +155,13 @@ def execute_test(profile, test): if len(rows) > 1: raise RuntimeError( "Bad test {name}: Returned {num_rows} rows instead of 1" - .format(name=model.name, num_rows=len(rows))) + .format(name=test.name, num_rows=len(rows))) row = rows[0] if len(row) > 1: raise RuntimeError( "Bad test {name}: Returned {num_cols} cols instead of 1" - .format(name=model.name, num_cols=len(row))) + .format(name=test.name, num_cols=len(row))) return row[0] diff --git a/dbt/utils.py b/dbt/utils.py index db5cb52db46..fe0fcfd2c2a 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -20,6 +20,15 @@ ] +class NodeType(object): + Base = 'base' + Model = 'model' + Analysis = 'analysis' + Test = 'test' + Archive = 'archive' + Macro = 'macro' + + class This(object): def __init__(self, schema, table, name): self.schema = schema @@ -109,21 +118,31 @@ def model_cte_name(model): return '__dbt__CTE__{}'.format(model.get('name')) -def find_model_by_name(flat_graph, target_model_name, - target_model_package): +def find_model_by_name(flat_graph, target_name, target_package): + return find_by_name(flat_graph, target_name, target_package, + 'nodes', NodeType.Model) + - for name, model in flat_graph.get('nodes').items(): - resource_type, package_name, model_name = name.split('.') +def find_macro_by_name(flat_graph, target_name, target_package): + return find_by_name(flat_graph, target_name, target_package, + 'macros', NodeType.Macro) - if (resource_type == 'model' and - ((target_model_name == model_name) and - (target_model_package is None or - target_model_package == package_name))): + +def find_by_name(flat_graph, target_name, target_package, subgraph, + nodetype): + for name, model in flat_graph.get(subgraph).items(): + resource_type, package_name, node_name = name.split('.') + + if (resource_type == nodetype and + ((target_name == node_name) and + (target_package is None or + target_package == package_name))): return model return None + def find_model_by_fqn(models, fqn): for model in models: if tuple(model.fqn) == tuple(fqn): diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index b32f3ebf712..89f500b444e 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -1176,7 +1176,7 @@ def test__simple_macro_used_in_model(self): 'package_name': 'root', 'root_path': get_os_path('/usr/src/app'), 'path': 'model_one.sql', - 'raw_sql': ("select *, {{root.simple(1, 2)}} from events"), + 'raw_sql': ("select *, {{package.simple(1, 2)}} from events"), }] self.assertEquals( @@ -1196,7 +1196,49 @@ def test__simple_macro_used_in_model(self): 'root_path': get_os_path('/usr/src/app'), 'depends_on': { 'nodes': [], - 'macros': [] + 'macros': [ + 'macro.package.simple' + ] + }, + 'config': self.model_config, + 'tags': set(), + 'path': 'model_one.sql', + 'raw_sql': self.find_input_by_name( + models, 'model_one').get('raw_sql') + } + } + ) + + def test__macro_no_explicit_project_used_in_model(self): + models = [{ + 'name': 'model_one', + 'resource_type': 'model', + 'package_name': 'root', + 'root_path': get_os_path('/usr/src/app'), + 'path': 'model_one.sql', + 'raw_sql': ("select *, {{ simple(1, 2) }} from events"), + }] + + self.assertEquals( + dbt.parser.parse_sql_nodes( + models, + self.root_project_config, + {'root': self.root_project_config, + 'snowplow': self.snowplow_project_config}), + { + 'model.root.model_one': { + 'name': 'model_one', + 'resource_type': 'model', + 'unique_id': 'model.root.model_one', + 'fqn': ['root', 'model_one'], + 'empty': False, + 'package_name': 'root', + 'root_path': get_os_path('/usr/src/app'), + 'depends_on': { + 'nodes': [], + 'macros': [ + 'macro.root.simple' + ] }, 'config': self.model_config, 'tags': set(), From a70ab42f9ec88a02039da673d4c8abb41ef4f27c Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 15 Mar 2017 16:10:05 -0400 Subject: [PATCH 05/10] add test for nested macros, refactor a bit --- dbt/clients/jinja.py | 8 ++- dbt/compilation.py | 72 ++++++++++--------- dbt/contracts/graph/parsed.py | 5 ++ dbt/parser.py | 35 ++++++--- .../016_macro_tests/macros/nested_macros.sql | 5 ++ .../016_macro_tests/models/nested_macro.sql | 9 +++ .../016_macro_tests/test_macros.py | 1 + test/unit/test_parser.py | 6 ++ 8 files changed, 96 insertions(+), 45 deletions(-) create mode 100644 test/integration/016_macro_tests/macros/nested_macros.sql create mode 100644 test/integration/016_macro_tests/models/nested_macro.sql diff --git a/dbt/clients/jinja.py b/dbt/clients/jinja.py index 3ad0153c872..9c7a092413e 100644 --- a/dbt/clients/jinja.py +++ b/dbt/clients/jinja.py @@ -62,11 +62,15 @@ def get_template(string, ctx, node=None, capture_macros=False): dbt.exceptions.raise_compiler_error(node, str(e)) -def get_rendered(string, ctx, node=None, capture_macros=False): +def render_template(template, ctx, node=None): try: - template = get_template(string, ctx, node, capture_macros) return template.render(ctx) except (jinja2.exceptions.TemplateSyntaxError, jinja2.exceptions.UndefinedError) as e: dbt.exceptions.raise_compiler_error(node, str(e)) + + +def get_rendered(string, ctx, node=None, capture_macros=False): + template = get_template(string, ctx, node, capture_macros) + return render_template(template, ctx, node=None) diff --git a/dbt/compilation.py b/dbt/compilation.py index 674ffde089f..f3168b6bd30 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -6,7 +6,6 @@ import dbt.utils from dbt.model import Model -from dbt.source import Source from dbt.utils import This, Var, is_enabled, get_materialization, NodeType from dbt.linker import Linker @@ -30,6 +29,44 @@ graph_file_name = 'graph.gpickle' +def recursively_parse_macros_for_node(node, flat_graph, context): + # this once worked, but is now long dead + # for unique_id in node.get('depends_on', {}).get('macros'): + + for unique_id, macro in flat_graph.get('macros').items(): + if macro is None: + dbt.exceptions.macro_not_found(node, unique_id) + + name = macro.get('name') + package_name = macro.get('package_name') + + if context.get(package_name, {}).get(name) is not None: + # we've already re-parsed this macro and added it to + # the context. + continue + + reparsed = dbt.parser.parse_macro_file( + macro_file_path=macro.get('path'), + macro_file_contents=macro.get('raw_sql'), + root_path=macro.get('root_path'), + package_name=package_name, + context=context) + + for unique_id, macro in reparsed.items(): + macro_map = {macro.get('name'): macro.get('parsed_macro')} + + if context.get(package_name) is None: + context[package_name] = {} + + context.get(package_name, {}) \ + .update(macro_map) + + if package_name == node.get('package_name'): + context.update(macro_map) + + return context + + def compile_and_print_status(project, args): compiler = Compiler(project, args) compiler.initialize() @@ -245,37 +282,8 @@ def get_compiler_context(self, linker, model, flat_graph): context['invocation_id'] = '{{ invocation_id }}' context['sql_now'] = adapter.date_function - for unique_id in model.get('depends_on', {}).get('macros'): - macro = flat_graph.get('macros', {}).get(unique_id) - - if macro is None: - dbt.exceptions.macro_not_found(model, unique_id) - - name = macro.get('name') - package_name = macro.get('package_name') - - if context.get(package_name, {}).get(name) is not None: - # we've already re-parsed this macro and added it to - # the context. - continue - - reparsed = dbt.parser.parse_macro_file( - macro_file_path=macro.get('path'), - macro_file_contents=macro.get('raw_sql'), - root_path=macro.get('root_path'), - package_name=package_name) - - for unique_id, macro in reparsed.items(): - macro_map = {macro.get('name'): macro.get('parsed_macro')} - - if context.get(package_name) is None: - context[package_name] = {} - - context.get(package_name, {}) \ - .update(macro_map) - - if package_name == model.get('package_name'): - context.update(macro_map) + context = recursively_parse_macros_for_node( + model, flat_graph, context) return context diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 3b2111cfe69..692075d1455 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -53,6 +53,11 @@ Required('unique_id'): All(basestring, Length(min=1, max=255)), Required('tags'): All(set), + # parsed fields + Required('depends_on'): { + Required('macros'): [All(basestring, Length(min=1, max=255))], + }, + # contents Required('parsed_macro'): jinja2.runtime.Macro diff --git a/dbt/parser.py b/dbt/parser.py index c521d33062a..18d3a3e5838 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -111,7 +111,8 @@ def parse_macro_file(macro_file_path, macro_file_contents, root_path, package_name, - tags=None): + tags=None, + context=None): logger.debug("Parsing {}".format(macro_file_path)) @@ -120,12 +121,24 @@ def parse_macro_file(macro_file_path, if tags is None: tags = set() - template = dbt.clients.jinja.get_template(macro_file_contents, { - 'ref': lambda *args: '', - 'var': lambda *args: '', - 'target': property(lambda x: '', lambda x: x), - 'this': '' - }) + if context is None: + context = { + 'ref': lambda *args: '', + 'var': lambda *args: '', + 'target': property(lambda x: '', lambda x: x), + 'this': '' + } + + base_node = { + 'resource_type': NodeType.Macro, + 'package_name': package_name, + 'depends_on': { + 'macros': [], + } + } + + template = dbt.clients.jinja.get_template( + macro_file_contents, context, node=base_node) for key, item in template.module.__dict__.items(): if type(item) == jinja2.runtime.Macro: @@ -133,17 +146,17 @@ def parse_macro_file(macro_file_path, package_name, key) - to_return[unique_id] = { + new_node = base_node.copy() + new_node.update({ 'name': key, 'unique_id': unique_id, 'tags': tags, - 'package_name': package_name, - 'resource_type': NodeType.Macro, 'root_path': root_path, 'path': macro_file_path, 'raw_sql': macro_file_contents, 'parsed_macro': item - } + }) + to_return[unique_id] = new_node return to_return diff --git a/test/integration/016_macro_tests/macros/nested_macros.sql b/test/integration/016_macro_tests/macros/nested_macros.sql new file mode 100644 index 00000000000..2096789696d --- /dev/null +++ b/test/integration/016_macro_tests/macros/nested_macros.sql @@ -0,0 +1,5 @@ +{% macro nested(arg1, arg2) %} + + {{ do_something2(arg1, arg2) }} + +{% endmacro %} diff --git a/test/integration/016_macro_tests/models/nested_macro.sql b/test/integration/016_macro_tests/models/nested_macro.sql new file mode 100644 index 00000000000..05acc7b7800 --- /dev/null +++ b/test/integration/016_macro_tests/models/nested_macro.sql @@ -0,0 +1,9 @@ +{{ + nested("arg1", "arg2") +}} + +union all + +{{ + test.do_something2("arg3", "arg4") +}} diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py index 84b0a592c47..fa9b910402a 100644 --- a/test/integration/016_macro_tests/test_macros.py +++ b/test/integration/016_macro_tests/test_macros.py @@ -36,6 +36,7 @@ def test_working_macros(self): self.assertTablesEqual("expected_dep_macro","dep_macro") self.assertTablesEqual("expected_local_macro","local_macro") + self.assertTablesEqual("expected_local_macro","nested_macro") class TestInvalidMacros(DBTIntegrationTest): diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 89f500b444e..68ce07b11cd 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -1127,6 +1127,9 @@ def test__simple_macro(self): 'resource_type': 'macro', 'unique_id': 'macro.root.simple', 'package_name': 'root', + 'depends_on': { + 'macros': [] + }, 'root_path': get_os_path('/usr/src/app'), 'tags': set(), 'path': 'simple_macro.sql', @@ -1162,6 +1165,9 @@ def test__simple_macro_used_in_model(self): 'resource_type': 'macro', 'unique_id': 'macro.root.simple', 'package_name': 'root', + 'depends_on': { + 'macros': [] + }, 'root_path': get_os_path('/usr/src/app'), 'tags': set(), 'path': 'simple_macro.sql', From 50a680ea434f980a7b013dc4b190d10f8f04da72 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 15 Mar 2017 16:12:13 -0400 Subject: [PATCH 06/10] use defaultdict for stats --- dbt/compilation.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dbt/compilation.py b/dbt/compilation.py index f3168b6bd30..a1f922b4257 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -1,5 +1,5 @@ import os -from collections import OrderedDict +from collections import OrderedDict, defaultdict import sqlparse import dbt.project @@ -579,14 +579,12 @@ def compile(self): self.write_graph_file(linker) - stats = {} + stats = defaultdict(int) for node_name, node in compiled_graph.get('nodes').items(): - stats[node.get('resource_type')] = stats.get( - node.get('resource_type'), 0) + 1 + stats[node.get('resource_type')] += 1 for node_name, node in compiled_graph.get('macros').items(): - stats[node.get('resource_type')] = stats.get( - node.get('resource_type'), 0) + 1 + stats[node.get('resource_type')] += 1 return stats From bec160c9380039994d5e4ad7508ef620e8fa8f9d Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 15 Mar 2017 16:25:12 -0400 Subject: [PATCH 07/10] tweakin --- dbt/compilation.py | 1 + dbt/exceptions.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/compilation.py b/dbt/compilation.py index a1f922b4257..aa0a6391996 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -32,6 +32,7 @@ def recursively_parse_macros_for_node(node, flat_graph, context): # this once worked, but is now long dead # for unique_id in node.get('depends_on', {}).get('macros'): + # TODO: make it so that we only parse the necessary macros for any node. for unique_id, macro in flat_graph.get('macros').items(): if macro is None: diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 5c341b14b73..39bf8bd170a 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -78,5 +78,5 @@ def dependency_not_found(model, target_model_name): def macro_not_found(model, target_macro_id): raise_compiler_error( model, - "'{}' references macro '{}' which is not in the graph!" + "'{}' references macro '{}' which is not defined!" .format(model.get('unique_id'), target_macro_id)) From be78232d33e13bbc658a774000da280e76dea5df Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 15 Mar 2017 16:45:30 -0400 Subject: [PATCH 08/10] pep8 --- dbt/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/utils.py b/dbt/utils.py index fe0fcfd2c2a..3a2de8e0714 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -142,7 +142,6 @@ def find_by_name(flat_graph, target_name, target_package, subgraph, return None - def find_model_by_fqn(models, fqn): for model in models: if tuple(model.fqn) == tuple(fqn): From 36c69defb45b197407dd3b5e32edbeafc43dd3d6 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 16 Mar 2017 13:41:23 -0400 Subject: [PATCH 09/10] actually fix pep8, remove nested macro. it does not work. --- dbt/contracts/graph/parsed.py | 2 +- .../016_macro_tests/macros/nested_macros.sql | 5 ----- .../016_macro_tests/models/nested_macro.sql | 9 --------- test/integration/016_macro_tests/test_macros.py | 15 +++++++++------ 4 files changed, 10 insertions(+), 21 deletions(-) delete mode 100644 test/integration/016_macro_tests/macros/nested_macros.sql delete mode 100644 test/integration/016_macro_tests/models/nested_macro.sql diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index 692075d1455..0b3342802d5 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -9,7 +9,7 @@ from dbt.contracts.graph.unparsed import unparsed_node_contract, \ unparsed_base_contract -from dbt.logger import GLOBAL_LOGGER as logger # noqa +from dbt.logger import GLOBAL_LOGGER as logger # noqa config_contract = { diff --git a/test/integration/016_macro_tests/macros/nested_macros.sql b/test/integration/016_macro_tests/macros/nested_macros.sql deleted file mode 100644 index 2096789696d..00000000000 --- a/test/integration/016_macro_tests/macros/nested_macros.sql +++ /dev/null @@ -1,5 +0,0 @@ -{% macro nested(arg1, arg2) %} - - {{ do_something2(arg1, arg2) }} - -{% endmacro %} diff --git a/test/integration/016_macro_tests/models/nested_macro.sql b/test/integration/016_macro_tests/models/nested_macro.sql deleted file mode 100644 index 05acc7b7800..00000000000 --- a/test/integration/016_macro_tests/models/nested_macro.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - nested("arg1", "arg2") -}} - -union all - -{{ - test.do_something2("arg3", "arg4") -}} diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py index fa9b910402a..59ec4b2c061 100644 --- a/test/integration/016_macro_tests/test_macros.py +++ b/test/integration/016_macro_tests/test_macros.py @@ -1,6 +1,7 @@ from nose.plugins.attrib import attr from test.integration.base import DBTIntegrationTest + class TestMacros(DBTIntegrationTest): def setUp(self): @@ -34,9 +35,8 @@ def test_working_macros(self): self.run_dbt(["deps"]) self.run_dbt(["run"]) - self.assertTablesEqual("expected_dep_macro","dep_macro") - self.assertTablesEqual("expected_local_macro","local_macro") - self.assertTablesEqual("expected_local_macro","nested_macro") + self.assertTablesEqual("expected_dep_macro", "dep_macro") + self.assertTablesEqual("expected_local_macro", "local_macro") class TestInvalidMacros(DBTIntegrationTest): @@ -63,10 +63,13 @@ def test_invalid_macro(self): try: self.run_dbt(["compile"]) - self.assertTrue(False, 'compiling bad macro should raise a runtime error') - except RuntimeError as e: + self.assertTrue(False, + 'compiling bad macro should raise a runtime error') + + except RuntimeError: pass + class TestMisusedMacros(DBTIntegrationTest): def setUp(self): @@ -96,5 +99,5 @@ def test_working_macros(self): try: self.run_dbt(["run"]) self.assertTrue(False, 'invoked a package macro from global scope') - except RuntimeError as e: + except RuntimeError: pass From 6289b3663fc55ad318077f6b2cc49491c11ef02b Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 17 Mar 2017 14:00:10 -0400 Subject: [PATCH 10/10] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a8de27ffcc..8cf3be48daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Graph refactor: fix common issues with load order ([#292](https://github.com/fishtown-analytics/dbt/pull/292)) - Graph refactor: multiple references to an ephemeral models should share a CTE ([#316](https://github.com/fishtown-analytics/dbt/pull/316)) +- Graph refactor: macros in flat graph ([#332](https://github.com/fishtown-analytics/dbt/pull/332)) - Refactor: factor out jinja interactions ([#309](https://github.com/fishtown-analytics/dbt/pull/309)) - Speedup: detect cycles at the end of compilation ([#307](https://github.com/fishtown-analytics/dbt/pull/307)) - Speedup: write graph file with gpickle instead of yaml ([#306](https://github.com/fishtown-analytics/dbt/pull/306))