diff --git a/dbt/clients/yaml_helper.py b/dbt/clients/yaml_helper.py new file mode 100644 index 00000000000..3e4c2ac2bbf --- /dev/null +++ b/dbt/clients/yaml_helper.py @@ -0,0 +1,57 @@ +import dbt.compat +import dbt.exceptions + +import yaml +import yaml.scanner + + +YAML_ERROR_MESSAGE = """ +Syntax error near line {line_number} +------------------------------ +{nice_error} + +Raw Error: +------------------------------ +{raw_error} +""".strip() + + +def line_no(i, line, width=3): + line_number = dbt.compat.to_string(i).ljust(width) + return "{}| {}".format(line_number, line) + + +def prefix_with_line_numbers(string, no_start, no_end): + line_list = string.split('\n') + + numbers = range(no_start, no_end) + relevant_lines = line_list[no_start:no_end] + + return "\n".join([ + line_no(i + 1, line) for (i, line) in zip(numbers, relevant_lines) + ]) + + +def contextualized_yaml_error(raw_contents, error): + mark = error.problem_mark + + min_line = max(mark.line - 3, 0) + max_line = mark.line + 4 + + nice_error = prefix_with_line_numbers(raw_contents, min_line, max_line) + + return YAML_ERROR_MESSAGE.format(line_number=mark.line + 1, + nice_error=nice_error, + raw_error=error) + + +def load_yaml_text(contents): + try: + return yaml.safe_load(contents) + except (yaml.scanner.ScannerError, yaml.YAMLError) as e: + if hasattr(e, 'problem_mark'): + error = contextualized_yaml_error(contents, e) + else: + error = dbt.compat.to_string(e) + + raise dbt.exceptions.ValidationException(error) diff --git a/dbt/config.py b/dbt/config.py index 32f769e02a9..639cb161caf 100644 --- a/dbt/config.py +++ b/dbt/config.py @@ -1,24 +1,30 @@ import os.path -import yaml -import yaml.scanner import dbt.exceptions +import dbt.clients.yaml_helper +import dbt.clients.system from dbt.logger import GLOBAL_LOGGER as logger +INVALID_PROFILE_MESSAGE = """ +dbt encountered an error while trying to read your profiles.yml file. + +{error_string} +""" + + def read_profile(profiles_dir): - # TODO: validate profiles_dir path = os.path.join(profiles_dir, 'profiles.yml') + contents = None if os.path.isfile(path): try: - with open(path, 'r') as f: - return yaml.safe_load(f) - except (yaml.scanner.ScannerError, - yaml.YAMLError) as e: - raise dbt.exceptions.ValidationException( - ' Could not read {}\n\n{}'.format(path, str(e))) + contents = dbt.clients.system.load_file_contents(path, strip=False) + return dbt.clients.yaml_helper.load_yaml_text(contents) + except dbt.exceptions.ValidationException as e: + msg = INVALID_PROFILE_MESSAGE.format(error_string=e) + raise dbt.exceptions.ValidationException(msg) return {} diff --git a/dbt/main.py b/dbt/main.py index 3238638461b..116e40d860d 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -20,6 +20,13 @@ import dbt.config as config import dbt.adapters.cache as adapter_cache import dbt.ui.printer +import dbt.compat + +PROFILES_HELP_MESSAGE = """ +For more information on configuring profiles, please consult the dbt docs: + +https://dbt.readme.io/docs/configure-your-profile +""" def main(args=None): @@ -139,17 +146,19 @@ def invoke_dbt(parsed): proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") - logger.info(" ERROR {}".format(str(e))) - logger.info( - "Did you set the correct --profile? Using: {}" - .format(parsed.profile)) - - logger.info("Valid profiles:") + logger.info(dbt.compat.to_string(e)) all_profiles = project.read_profiles(parsed.profiles_dir).keys() - for profile in all_profiles: - logger.info(" - {}".format(profile)) + if len(all_profiles) > 0: + logger.info("Defined profiles:") + for profile in all_profiles: + logger.info(" - {}".format(profile)) + else: + logger.info("There are no profiles defined in your " + "profiles.yml file") + + logger.info(PROFILES_HELP_MESSAGE) dbt.tracking.track_invalid_invocation( project=proj, diff --git a/dbt/parser.py b/dbt/parser.py index f1ba1c21e3b..8451ff0c6f2 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -1,6 +1,5 @@ import copy import os -import yaml import re import dbt.flags @@ -9,6 +8,7 @@ import jinja2.runtime import dbt.clients.jinja +import dbt.clients.yaml_helper import dbt.contracts.graph.parsed import dbt.contracts.graph.unparsed @@ -426,7 +426,14 @@ def parse_schema_tests(tests, root_project, projects): to_return = {} for test in tests: - test_yml = yaml.safe_load(test.get('raw_yml')) + raw_yml = test.get('raw_yml') + test_name = "{}:{}".format(test.get('package_name'), test.get('path')) + + try: + test_yml = dbt.clients.yaml_helper.load_yaml_text(raw_yml) + except dbt.exceptions.ValidationException as e: + test_yml = None + logger.info("Error reading {} - Skipping\n{}".format(test_name, e)) if test_yml is None: continue @@ -551,7 +558,7 @@ def load_and_parse_yml(package_name, root_project, all_projects, root_dir, for file_match in file_matches: file_contents = dbt.clients.system.load_file_contents( - file_match.get('absolute_path')) + file_match.get('absolute_path'), strip=False) parts = dbt.utils.split_path(file_match.get('relative_path', '')) name, _ = os.path.splitext(parts[-1]) diff --git a/dbt/project.py b/dbt/project.py index 77a81a0b16a..8d66a0ee249 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -1,14 +1,13 @@ import os.path -import yaml import pprint import copy -import sys import hashlib import re -from voluptuous import Schema, Required, Invalid +from voluptuous import Required, Invalid import dbt.deprecations import dbt.contracts.connection +import dbt.clients.yaml_helper from dbt.logger import GLOBAL_LOGGER as logger default_project_cfg = { @@ -30,6 +29,19 @@ default_profiles_dir = os.path.join(os.path.expanduser('~'), '.dbt') +NO_SUPPLIED_PROFILE_ERROR = """\ +dbt cannot run because no profile was specified for this dbt project. +To specify a profile for this project, add a line like the this to +your dbt_project.yml file: + +profile: [profile name] + +Here, [profile name] should be replaced with a profile name +defined in your profiles.yml file. You can find profiles.yml here: + +{profiles_file}/profiles.yml +""".format(profiles_file=default_profiles_dir) + class DbtProjectError(Exception): def __init__(self, message, project): @@ -60,9 +72,7 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None, self.profile_to_load = self.cfg['profile'] if self.profile_to_load is None: - raise DbtProjectError( - "No profile was supplied in the dbt_project.yml file, or the " - "command line", self) + raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR, self) if self.profile_to_load in self.profiles: self.cfg.update(self.profiles[self.profile_to_load]) @@ -187,7 +197,7 @@ def read_project(filename, profiles_dir=None, validate=True, project_file_contents = dbt.clients.system.load_file_contents(filename) - project_cfg = yaml.safe_load(project_file_contents) + project_cfg = dbt.clients.yaml_helper.load_yaml_text(project_file_contents) project_cfg['project-root'] = os.path.dirname( os.path.abspath(filename)) profiles = read_profiles(profiles_dir)