Improve CompilationException messaging for generic test Jinja renderi…

…ng (dbt-labs#5393)

* feat: Improve generic test UndefinedMacroException message

The error message rendered from the `UndefinedMacroException` when
raised by a TestBuilder is very vague as to where the problem is
and how to resolve it. This commit adds a basic amount of
information about the specific model and column that is
referencing an undefined macro.

Note: All custom macros referenced in a generic test config will
raise an UndefinedMacroException as of v0.20.0.

* feat: Bubble CompilationException into schemas.py

I realized that this exception information would be better if
CompilationExceptions inclulded the file that raised the exception.
To that end, I created a new exception handler in `_parse_generic_test`
to report on CompilationExceptions raised during the parsing of
generic tests. Along the way I reformatted the message returned
from TestBuilder to play nicely with the the existing formatting of
`_parse_generic_test`'s exception handling code.

* feat: Add tests to confirm CompileException

I've added a basic test to confirm that the approriate
CompilationException when a custom macro is referenced
in a generic test config.

* feat: Add changie entry and tweak error msg

* Update .changes/unreleased/Under the Hood-20220617-150744.yaml

Thanks to @emmyoop for the recommendation that this be listed as a Fix change instead of an "Under the Hood" change!

Co-authored-by: Emily Rockman <emily.rockman@dbtlabs.com>

* fix: Simplified Compliation Error message

I've simplified the error message raised during a Compilation Error
sourced from a test config. Mainly by way of removing tabs and newlines
where not required.

* fix: Convert format to fstring in schemas

This commit moves a format call to a multiline fstring in the
schemas.py file for CompilationExceptions.

Co-authored-by: Emily Rockman <emily.rockman@dbtlabs.com>

.changes/unreleased/Under the Hood-20220617-150744.yaml

@@ -0,0 +1,8 @@
+kind: Fixes
+body: Add context to compilation errors generated while rendering generic test configuration
+ values.
+time: 2022-06-17T15:07:44.751037-04:00
+custom:
+ Author: nicholasyager
+ Issue: "5294"
+ PR: "5393"

core/dbt/parser/generic_test_builders.py

@@ -21,7 +21,7 @@
  UnparsedNodeUpdate,
  UnparsedExposure,
 )
-from dbt.exceptions import raise_compiler_error, raise_parsing_error
+from dbt.exceptions import raise_compiler_error, raise_parsing_error, UndefinedMacroException
 from dbt.parser.search import FileBlock
 
 
@@ -257,7 +257,28 @@ def __init__(
  if not value and "config" in self.args:
  value = self.args["config"].pop(key, None)
  if isinstance(value, str):
- value = get_rendered(value, render_ctx, native=True)
+
+ try:
+ value = get_rendered(value, render_ctx, native=True)
+ except UndefinedMacroException as e:
+
+ # Generic tests do not include custom macros in the Jinja
+ # rendering context, so this will almost always fail. As it
+ # currently stands, the error message is inscrutable, which
+ # has caused issues for some projects migrating from
+ # pre-0.20.0 to post-0.20.0.
+ # See https://github.com/dbt-labs/dbt-core/issues/4103
+ # and https://github.com/dbt-labs/dbt-core/issues/5294
+ raise_compiler_error(
+ f"The {self.target.name}.{column_name} column's "
+ f'"{self.name}" test references an undefined '
+ f"macro in its {key} configuration argument. "
+ f"The macro {e.msg}.\n"
+ "Please note that the generic test configuration parser "
+ "currently does not support using custom macros to "
+ "populate configuration values"
+ )
+
  if value is not None:
  self.config[key] = value
 

core/dbt/parser/schemas.py

@@ -57,6 +57,7 @@
  InternalException,
  raise_duplicate_source_patch_name,
  warn_or_error,
+ CompilationException,
 )
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleParser
@@ -302,6 +303,15 @@ def _parse_generic_test(
  target.original_file_path, exc.msg, context
  )
  raise ParsingException(msg) from exc
+
+ except CompilationException as exc:
+ context = _trimmed(str(target))
+ msg = (
+ "Invalid generic test configuration given in "
+ f"{target.original_file_path}: \n{exc.msg}\n\t@: {context}"
+ )
+ raise CompilationException(msg) from exc
+
  original_name = os.path.basename(target.original_file_path)
  compiled_path = get_pseudo_test_path(builder.compiled_name, original_name)
 

tests/functional/schema_tests/fixtures.py

@@ -442,6 +442,24 @@
 """
 
 
+custom_generic_test_config_custom_macro__schema_yml = """
+version: 2
+models:
+- name: model_a
+ columns:
+ - name: id
+ tests:
+ - not_null:
+ config:
+ where: "id = (select id from {{ ref('model_a') }} limit 1)"
+
+"""
+
+custom_generic_test_config_custom_macro__model_a = """
+SELECT 1 AS id
+"""
+
+
 custom_generic_test_names__schema_yml = """
 version: 2
 models:
@@ -1341,6 +1359,14 @@ def dupe_tests_collide():
  }
 
 
+@pytest.fixture(scope="class")
+def custom_generic_test_config_custom_macros():
+ return {
+ "schema.yml": custom_generic_test_config_custom_macro__schema_yml,
+ "model_a.sql": custom_generic_test_config_custom_macro__model_a,
+ }
+
+
 @pytest.fixture(scope="class")
 def custom_generic_test_names():
  return {

tests/functional/schema_tests/test_schema_v2_tests.py

@@ -17,6 +17,7 @@
  test_context_models,
  name_collision,
  dupe_tests_collide,
+ custom_generic_test_config_custom_macros,
  custom_generic_test_names,
  custom_generic_test_names_alt_format,
  test_context_where_subq_macros,
@@ -676,6 +677,21 @@ def test_generic_test_collision(
  assert "dbt found two tests with the name" in str(exc)
 
 
+class TestGenericTestsConfigCustomMacros:
+ @pytest.fixture(scope="class")
+ def models(self, custom_generic_test_config_custom_macros): # noqa: F811
+ return custom_generic_test_config_custom_macros
+
+ def test_generic_test_config_custom_macros(
+ self,
+ project,
+ ):
+ """This test has a reference to a custom macro its configs"""
+ with pytest.raises(CompilationException) as exc:
+ run_dbt()
+ assert "Invalid generic test configuration" in str(exc)
+
+
 class TestGenericTestsCustomNames:
  @pytest.fixture(scope="class")
  def models(self, custom_generic_test_names): # noqa: F811