From 37d32037e2e418fdeb4109ae263dffadb64c886e Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 7 Dec 2017 16:46:32 -0500 Subject: [PATCH 1/3] skip tests that depend on nonexistent or disabled models --- dbt/exceptions.py | 17 ++++++++++------- dbt/parser.py | 7 ++++++- .../001_simple_copy_test/models/schema.yml | 8 ++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 test/integration/001_simple_copy_test/models/schema.yml diff --git a/dbt/exceptions.py b/dbt/exceptions.py index d38af8605ac..d135c86b301 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -1,4 +1,5 @@ from dbt.compat import basestring +from dbt.logger import GLOBAL_LOGGER as logger class Exception(BaseException): @@ -146,18 +147,20 @@ def ref_bad_context(model, target_model_name, target_model_package): raise_compiler_error(error_msg, model) -def ref_target_not_found(model, target_model_name, target_model_package): +def ref_target_not_found(model, target_model_name, target_model_package, warn=False): target_package_string = '' if target_model_package is not None: target_package_string = "in package '{}' ".format(target_model_package) - raise_compiler_error( - "Model '{}' depends on model '{}' {}which was not found." - .format(model.get('unique_id'), - target_model_name, - target_package_string), - model) + msg = ("Model '{}' depends on model '{}' {}which was not found or is" + " disabled".format(model.get('unique_id'), + target_model_name, + target_package_string)) + if warn: + logger.debug("WARNING: {}".format(msg)) + else: + raise_compiler_error(msg, model) def ref_disabled_dependency(model, target_model): diff --git a/dbt/parser.py b/dbt/parser.py index b3c602650df..53637d5495b 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -91,11 +91,16 @@ def process_refs(flat_graph, current_project): current_project, node.get('package_name')) + node_is_test = (node.get('resource_type') == NodeType.Test) if target_model is None: dbt.exceptions.ref_target_not_found( node, target_model_name, - target_model_package) + target_model_package, + warn=node_is_test) + + if node_is_test: + continue if (dbt.utils.is_enabled(node) and not dbt.utils.is_enabled(target_model)): diff --git a/test/integration/001_simple_copy_test/models/schema.yml b/test/integration/001_simple_copy_test/models/schema.yml new file mode 100644 index 00000000000..f8eb708f45f --- /dev/null +++ b/test/integration/001_simple_copy_test/models/schema.yml @@ -0,0 +1,8 @@ + + +# Confirm that this does not throw an exception for +# a missing ref to the disabled model +disabled: + constraints: + unique: + - id From b10a99d139465f99e10b7285d1b211b5e7ba4d49 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 7 Dec 2017 18:21:54 -0500 Subject: [PATCH 2/3] pep8, Fixes https://github.com/fishtown-analytics/dbt/issues/616 --- dbt/exceptions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index d135c86b301..2082dcd878c 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -147,7 +147,8 @@ def ref_bad_context(model, target_model_name, target_model_package): raise_compiler_error(error_msg, model) -def ref_target_not_found(model, target_model_name, target_model_package, warn=False): +def ref_target_not_found(model, target_model_name, target_model_package, + warn=False): target_package_string = '' if target_model_package is not None: From 275e10eb04a67b6f0eec8821d633914cd2d23826 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 18 Dec 2017 12:24:17 -0500 Subject: [PATCH 3/3] refactor --- dbt/exceptions.py | 21 +++++++++++---------- dbt/parser.py | 23 +++++++---------------- dbt/utils.py | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 2082dcd878c..56c65928b91 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -147,21 +147,22 @@ def ref_bad_context(model, target_model_name, target_model_package): raise_compiler_error(error_msg, model) -def ref_target_not_found(model, target_model_name, target_model_package, - warn=False): +def get_target_not_found_msg(model, target_model_name, target_model_package): target_package_string = '' if target_model_package is not None: target_package_string = "in package '{}' ".format(target_model_package) - msg = ("Model '{}' depends on model '{}' {}which was not found or is" - " disabled".format(model.get('unique_id'), - target_model_name, - target_package_string)) - if warn: - logger.debug("WARNING: {}".format(msg)) - else: - raise_compiler_error(msg, model) + return ("Model '{}' depends on model '{}' {}which was not found or is" + " disabled".format(model.get('unique_id'), + target_model_name, + target_package_string)) + + +def ref_target_not_found(model, target_model_name, target_model_package): + msg = get_target_not_found_msg(model, target_model_name, + target_model_package) + raise_compiler_error(msg, model) def ref_disabled_dependency(model, target_model): diff --git a/dbt/parser.py b/dbt/parser.py index 53637d5495b..128d296fb8a 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -91,23 +91,14 @@ def process_refs(flat_graph, current_project): current_project, node.get('package_name')) - node_is_test = (node.get('resource_type') == NodeType.Test) if target_model is None: - dbt.exceptions.ref_target_not_found( - node, - target_model_name, - target_model_package, - warn=node_is_test) - - if node_is_test: - continue - - if (dbt.utils.is_enabled(node) and not - dbt.utils.is_enabled(target_model)): - if dbt.utils.is_type(node, NodeType.Model): - dbt.exceptions.ref_disabled_dependency(node, target_model) - else: - node.get('config', {})['enabled'] = False + # This may raise. Even if it doesn't, we don't want to add + # this node to the graph b/c there is no destination node + node.get('config', {})['enabled'] = False + dbt.utils.invalid_ref_fail_unless_test(node, + target_model_name, + target_model_package) + continue target_model_id = target_model.get('unique_id') diff --git a/dbt/utils.py b/dbt/utils.py index 16fb1e585a8..44863d3ac74 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -359,3 +359,18 @@ def get_hashed_contents(model): def flatten_nodes(dep_list): return list(itertools.chain.from_iterable(dep_list)) + + +def invalid_ref_fail_unless_test(node, target_model_name, + target_model_package): + if node.get('resource_type') == NodeType.Test: + warning = dbt.exceptions.get_target_not_found_msg( + node, + target_model_name, + target_model_package) + logger.debug("WARNING: {}".format(warning)) + else: + dbt.exceptions.ref_target_not_found( + node, + target_model_name, + target_model_package)