Better error messages (#445)

* wip

* overhauled yaml validation error messages

* use yaml client (mostly) everywhere

* fix imports

* fix yaml client namespace for python2

* pep8

* code cleanup + typos

dbt/clients/yaml_helper.py

@@ -0,0 +1,57 @@
+import dbt.compat
+import dbt.exceptions
+
+import yaml
+import yaml.scanner
+
+
+YAML_ERROR_MESSAGE = """
+Syntax error near line {line_number}
+------------------------------
+{nice_error}
+
+Raw Error:
+------------------------------
+{raw_error}
+""".strip()
+
+
+def line_no(i, line, width=3):
+    line_number = dbt.compat.to_string(i).ljust(width)
+    return "{}| {}".format(line_number, line)
+
+
+def prefix_with_line_numbers(string, no_start, no_end):
+    line_list = string.split('\n')
+
+    numbers = range(no_start, no_end)
+    relevant_lines = line_list[no_start:no_end]
+
+    return "\n".join([
+        line_no(i + 1, line) for (i, line) in zip(numbers, relevant_lines)
+    ])
+
+
+def contextualized_yaml_error(raw_contents, error):
+    mark = error.problem_mark
+
+    min_line = max(mark.line - 3, 0)
+    max_line = mark.line + 4
+
+    nice_error = prefix_with_line_numbers(raw_contents, min_line, max_line)
+
+    return YAML_ERROR_MESSAGE.format(line_number=mark.line + 1,
+                                     nice_error=nice_error,
+                                     raw_error=error)
+
+
+def load_yaml_text(contents):
+    try:
+        return yaml.safe_load(contents)
+    except (yaml.scanner.ScannerError, yaml.YAMLError) as e:
+        if hasattr(e, 'problem_mark'):
+            error = contextualized_yaml_error(contents, e)
+        else:
+            error = dbt.compat.to_string(e)
+
+        raise dbt.exceptions.ValidationException(error)

dbt/config.py

@@ -1,24 +1,30 @@
 import os.path
-import yaml
-import yaml.scanner
 
 import dbt.exceptions
+import dbt.clients.yaml_helper
+import dbt.clients.system
 
 from dbt.logger import GLOBAL_LOGGER as logger
 
 
+INVALID_PROFILE_MESSAGE = """
+dbt encountered an error while trying to read your profiles.yml file.
+
+{error_string}
+"""
+
+
 def read_profile(profiles_dir):
-    # TODO: validate profiles_dir
     path = os.path.join(profiles_dir, 'profiles.yml')
 
+    contents = None
     if os.path.isfile(path):
         try:
-            with open(path, 'r') as f:
-                return yaml.safe_load(f)
-        except (yaml.scanner.ScannerError,
-                yaml.YAMLError) as e:
-            raise dbt.exceptions.ValidationException(
-                '  Could not read {}\n\n{}'.format(path, str(e)))
+            contents = dbt.clients.system.load_file_contents(path, strip=False)
+            return dbt.clients.yaml_helper.load_yaml_text(contents)
+        except dbt.exceptions.ValidationException as e:
+            msg = INVALID_PROFILE_MESSAGE.format(error_string=e)
+            raise dbt.exceptions.ValidationException(msg)
 
     return {}
 

dbt/main.py

@@ -20,6 +20,13 @@
 import dbt.config as config
 import dbt.adapters.cache as adapter_cache
 import dbt.ui.printer
+import dbt.compat
+
+PROFILES_HELP_MESSAGE = """
+For more information on configuring profiles, please consult the dbt docs:
+
+https://dbt.readme.io/docs/configure-your-profile
+"""
 
 
 def main(args=None):
@@ -143,17 +150,19 @@ def invoke_dbt(parsed):
         proj.validate()
     except project.DbtProjectError as e:
         logger.info("Encountered an error while reading the project:")
-        logger.info("  ERROR {}".format(str(e)))
-        logger.info(
-            "Did you set the correct --profile? Using: {}"
-            .format(parsed.profile))
-
-        logger.info("Valid profiles:")
+        logger.info(dbt.compat.to_string(e))
 
         all_profiles = project.read_profiles(parsed.profiles_dir).keys()
 
-        for profile in all_profiles:
-            logger.info(" - {}".format(profile))
+        if len(all_profiles) > 0:
+            logger.info("Defined profiles:")
+            for profile in all_profiles:
+                logger.info(" - {}".format(profile))
+        else:
+            logger.info("There are no profiles defined in your "
+                        "profiles.yml file")
+
+        logger.info(PROFILES_HELP_MESSAGE)
 
         dbt.tracking.track_invalid_invocation(
             project=proj,

dbt/parser.py

@@ -1,6 +1,5 @@
 import copy
 import os
-import yaml
 import re
 
 import dbt.flags
@@ -9,6 +8,7 @@
 
 import jinja2.runtime
 import dbt.clients.jinja
+import dbt.clients.yaml_helper
 
 import dbt.contracts.graph.parsed
 import dbt.contracts.graph.unparsed
@@ -426,7 +426,14 @@ def parse_schema_tests(tests, root_project, projects):
     to_return = {}
 
     for test in tests:
-        test_yml = yaml.safe_load(test.get('raw_yml'))
+        raw_yml = test.get('raw_yml')
+        test_name = "{}:{}".format(test.get('package_name'), test.get('path'))
+
+        try:
+            test_yml = dbt.clients.yaml_helper.load_yaml_text(raw_yml)
+        except dbt.exceptions.ValidationException as e:
+            test_yml = None
+            logger.info("Error reading {} - Skipping\n{}".format(test_name, e))
 
         if test_yml is None:
             continue
@@ -551,7 +558,7 @@ def load_and_parse_yml(package_name, root_project, all_projects, root_dir,
 
     for file_match in file_matches:
         file_contents = dbt.clients.system.load_file_contents(
-            file_match.get('absolute_path'))
+            file_match.get('absolute_path'), strip=False)
 
         parts = dbt.utils.split_path(file_match.get('relative_path', ''))
         name, _ = os.path.splitext(parts[-1])

dbt/project.py

@@ -1,14 +1,13 @@
 import os.path
-import yaml
 import pprint
 import copy
-import sys
 import hashlib
 import re
-from voluptuous import Schema, Required, Invalid
+from voluptuous import Required, Invalid
 
 import dbt.deprecations
 import dbt.contracts.connection
+import dbt.clients.yaml_helper
 from dbt.logger import GLOBAL_LOGGER as logger
 
 default_project_cfg = {
@@ -30,6 +29,19 @@
 
 default_profiles_dir = os.path.join(os.path.expanduser('~'), '.dbt')
 
+NO_SUPPLIED_PROFILE_ERROR = """\
+dbt cannot run because no profile was specified for this dbt project.
+To specify a profile for this project, add a line like the this to
+your dbt_project.yml file:
+
+profile: [profile name]
+
+Here, [profile name] should be replaced with a profile name
+defined in your profiles.yml file. You can find profiles.yml here:
+
+{profiles_file}/profiles.yml
+""".format(profiles_file=default_profiles_dir)
+
 
 class DbtProjectError(Exception):
     def __init__(self, message, project):
@@ -60,9 +72,7 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None,
             self.profile_to_load = self.cfg['profile']
 
         if self.profile_to_load is None:
-            raise DbtProjectError(
-                "No profile was supplied in the dbt_project.yml file, or the "
-                "command line", self)
+            raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR, self)
 
         if self.profile_to_load in self.profiles:
             self.cfg.update(self.profiles[self.profile_to_load])
@@ -187,7 +197,7 @@ def read_project(filename, profiles_dir=None, validate=True,
 
     project_file_contents = dbt.clients.system.load_file_contents(filename)
 
-    project_cfg = yaml.safe_load(project_file_contents)
+    project_cfg = dbt.clients.yaml_helper.load_yaml_text(project_file_contents)
     project_cfg['project-root'] = os.path.dirname(
         os.path.abspath(filename))
     profiles = read_profiles(profiles_dir)