From 8d32c870fcced648b85ae55f5e97cd098bc657f5 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 29 Mar 2019 08:15:12 -0600 Subject: [PATCH 1/2] attach nodes to exceptions and pass compiled sql into the error messages --- core/dbt/exceptions.py | 18 ++++++++++++++++-- core/dbt/node_runners.py | 2 ++ .../042_sources_test/test_sources.py | 10 +++++----- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index ae1fecdd1a6..22462bd5827 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -95,6 +95,17 @@ def __str__(self, prefix="! "): return lines[0] + "\n" + "\n".join( [" " + line for line in lines[1:]]) + def data(self): + result = Exception.data(self) + if self.node is None: + return result + + result.update({ + 'raw_sql': self.node.get('raw_sql'), + 'compiled_sql': self.node.get('injected_sql'), + }) + return result + class RPCFailureResult(RuntimeException): CODE = 10002 @@ -106,13 +117,16 @@ class RPCTimeoutException(RuntimeException): MESSAGE = 'RPC timeout error' def __init__(self, timeout): + super(RPCTimeoutException, self).__init__(self.MESSAGE) self.timeout = timeout def data(self): - return { + result = super(RPCTimeoutException, self).data() + result.update({ 'timeout': self.timeout, 'message': 'RPC timed out after {}s'.format(self.timeout), - } + }) + return result class DatabaseException(RuntimeException): diff --git a/core/dbt/node_runners.py b/core/dbt/node_runners.py index e695ea162df..85e580c91b9 100644 --- a/core/dbt/node_runners.py +++ b/core/dbt/node_runners.py @@ -526,6 +526,8 @@ def __init__(self, config, adapter, node, node_index, num_nodes): def handle_exception(self, e, ctx): if isinstance(e, dbt.exceptions.Exception): + if hasattr(e, 'node'): + e.node = ctx.node return rpc.dbt_error(e) elif isinstance(e, rpc.RPCException): return e diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index 3edf4641d7c..5bb5dab7f08 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -416,7 +416,7 @@ def assertSuccessfulRunResult(self, data, raw_sql, compiled_sql=None, table=None self.assertResultHasTimings(result, 'compile', 'execute') @use_profile('postgres') - def test_compile(self): + def test_compile_postgres(self): trivial = self.query( 'compile', 'select 1 as id', @@ -500,7 +500,7 @@ def test_compile(self): ) @use_profile('postgres') - def test_run(self): + def test_run_postgres(self): # seed + run dbt to make models before using them! self.run_dbt_with_vars(['seed']) self.run_dbt_with_vars(['run']) @@ -623,7 +623,7 @@ def test_run(self): ) @use_profile('postgres') - def test_invalid_requests(self): + def test_invalid_requests_postgres(self): data = self.query( 'xxxxxnotamethodxxxxx', 'hi this is not sql' @@ -660,7 +660,7 @@ def test_invalid_requests(self): self.assertEqual(error_data['type'], 'DatabaseException') self.assertEqual( error_data['message'], - 'Database Error\n syntax error at or near "hi"\n LINE 1: hi this is not sql\n ^' + 'Database Error in rpc foo (from remote system)\n syntax error at or near "hi"\n LINE 1: hi this is not sql\n ^' ) self.assertIn('logs', error_data) self.assertTrue(len(error_data['logs']) > 0) @@ -677,7 +677,7 @@ def test_invalid_requests(self): self.assertEqual(error_data['type'], 'DatabaseException') @use_profile('postgres') - def test_timeout(self): + def test_timeout_postgres(self): data = self.query( 'run', 'select from pg_sleep(5)', From ed59bd22f31245fdc8988661ccba0c9e300fa0f1 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 29 Mar 2019 09:03:17 -0600 Subject: [PATCH 2/2] actually override macros... fix macro overrides fix tests that masked the broken macro overrides --- core/dbt/task/compile.py | 3 +- .../042_sources_test/macros/macro.sql | 2 +- .../042_sources_test/test_sources.py | 59 +++++++++++-------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py index b56ff33b190..8b177812bb5 100644 --- a/core/dbt/task/compile.py +++ b/core/dbt/task/compile.py @@ -38,7 +38,7 @@ class RemoteCompileTask(CompileTask, RemoteCallable): def __init__(self, args, config, manifest): super(RemoteCompileTask, self).__init__(args, config) - self._base_manifest = manifest + self._base_manifest = manifest.deepcopy(config=config) def get_runner_type(self): return RPCCompileRunner @@ -82,6 +82,7 @@ def _get_exec_node(self, name, sql, macros): resource_type=NodeType.Macro )) + self._base_manifest.macros.update(macro_overrides) rpc_parser = RPCCallParser( self.config, all_projects=all_projects, diff --git a/test/integration/042_sources_test/macros/macro.sql b/test/integration/042_sources_test/macros/macro.sql index c1d3b1f47bb..a400a94f625 100644 --- a/test/integration/042_sources_test/macros/macro.sql +++ b/test/integration/042_sources_test/macros/macro.sql @@ -1,5 +1,5 @@ {% macro override_me() -%} - exceptions.raise_compiler_error('this is a bad macro') + {{ exceptions.raise_compiler_error('this is a bad macro') }} {%- endmacro %} {% macro happy_little_macro() -%} diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index 5bb5dab7f08..dd309cfb69e 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -389,6 +389,16 @@ def assertIsErrorWithCode(self, data, code): self.assertEqual(error['code'], code) return error + def assertIsErrorWith(self, data, code, message, error_data): + error = self.assertIsErrorWithCode(data, code) + if message is not None: + self.assertEqual(error['message'], message) + + if error_data is not None: + return self.assertHasErrorData(error, error_data) + else: + return error.get('data') + def assertResultHasSql(self, data, raw_sql, compiled_sql=None): if compiled_sql is None: compiled_sql = raw_sql @@ -628,23 +638,19 @@ def test_invalid_requests_postgres(self): 'xxxxxnotamethodxxxxx', 'hi this is not sql' ).json() - error = self.assertIsErrorWithCode(data, -32601) - self.assertEqual(error['message'], 'Method not found') + self.assertIsErrorWith(data, -32601, 'Method not found', None) data = self.query( 'compile', 'select * from {{ reff("nonsource_descendant") }}', name='mymodel' ).json() - error = self.assertIsErrorWithCode(data, 10004) - self.assertEqual(error['message'], 'Compilation Error') - self.assertIn('data', error) - error_data = error['data'] - self.assertEqual(error_data['type'], 'CompilationException') - self.assertEqual( - error_data['message'], - "Compilation Error in rpc mymodel (from remote system)\n 'reff' is undefined" - ) + error_data = self.assertIsErrorWith(data, 10004, 'Compilation Error', { + 'type': 'CompilationException', + 'message': "Compilation Error in rpc mymodel (from remote system)\n 'reff' is undefined", + 'compiled_sql': None, + 'raw_sql': 'select * from {{ reff("nonsource_descendant") }}', + }) self.assertIn('logs', error_data) self.assertTrue(len(error_data['logs']) > 0) @@ -653,15 +659,12 @@ def test_invalid_requests_postgres(self): 'hi this is not sql', name='foo' ).json() - error = self.assertIsErrorWithCode(data, 10003) - self.assertEqual(error['message'], 'Database Error') - self.assertIn('data', error) - error_data = error['data'] - self.assertEqual(error_data['type'], 'DatabaseException') - self.assertEqual( - error_data['message'], - 'Database Error in rpc foo (from remote system)\n syntax error at or near "hi"\n LINE 1: hi this is not sql\n ^' - ) + error_data = self.assertIsErrorWith(data, 10003, 'Database Error', { + 'type': 'DatabaseException', + 'message': 'Database Error in rpc foo (from remote system)\n syntax error at or near "hi"\n LINE 1: hi this is not sql\n ^', + 'compiled_sql': 'hi this is not sql', + 'raw_sql': 'hi this is not sql', + }) self.assertIn('logs', error_data) self.assertTrue(len(error_data['logs']) > 0) @@ -670,11 +673,21 @@ def test_invalid_requests_postgres(self): 'select {{ happy_little_macro() }}', name='foo', ).json() - self.assertIsErrorWithCode(macro_no_override, 10003) - self.assertEqual(error['message'], 'Database Error') + error_data = self.assertIsErrorWith(macro_no_override, 10004, 'Compilation Error', { + 'type': 'CompilationException', + 'raw_sql': 'select {{ happy_little_macro() }}', + 'compiled_sql': None + }) + self.assertIn('logs', error_data) + self.assertTrue(len(error_data['logs']) > 0) + + def assertHasErrorData(self, error, expected_error_data): self.assertIn('data', error) error_data = error['data'] - self.assertEqual(error_data['type'], 'DatabaseException') + for key, value in expected_error_data.items(): + self.assertIn(key, error_data) + self.assertEqual(error_data[key], value) + return error_data @use_profile('postgres') def test_timeout_postgres(self):