From d4c2dfedb271679226249b15c1b22f148d91af65 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 4 Dec 2018 15:01:11 -0700 Subject: [PATCH] dbt debug --- dbt/config.py | 12 +-- dbt/task/debug.py | 268 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 250 insertions(+), 30 deletions(-) diff --git a/dbt/config.py b/dbt/config.py index 843467c2fc0..af6e471167f 100644 --- a/dbt/config.py +++ b/dbt/config.py @@ -543,7 +543,7 @@ def _credentials_from_profile(profile, profile_name, target_name): return credentials @staticmethod - def _pick_profile_name(args_profile_name, project_profile_name=None): + def pick_profile_name(args_profile_name, project_profile_name=None): profile_name = project_profile_name if args_profile_name is not None: profile_name = args_profile_name @@ -606,8 +606,8 @@ def from_credentials(cls, credentials, threads, profile_name, target_name, return profile @classmethod - def _render_profile(cls, raw_profile, profile_name, target_override, - cli_vars): + def render_profile(cls, raw_profile, profile_name, target_override, + cli_vars): """This is a containment zone for the hateful way we're rendering profiles. """ @@ -664,7 +664,7 @@ def from_raw_profile_info(cls, raw_profile, profile_name, cli_vars, """ # user_cfg is not rendered since it only contains booleans. # TODO: should it be, and the values coerced to bool? - target_name, profile_data = cls._render_profile( + target_name, profile_data = cls.render_profile( raw_profile, profile_name, target_override, cli_vars ) @@ -749,8 +749,8 @@ def from_args(cls, args, project_profile_name=None, cli_vars=None): profiles_dir = getattr(args, 'profiles_dir', PROFILES_DIR) target_override = getattr(args, 'target', None) raw_profiles = read_profile(profiles_dir) - profile_name = cls._pick_profile_name(args.profile, - project_profile_name) + profile_name = cls.pick_profile_name(args.profile, + project_profile_name) return cls.from_raw_profiles( raw_profiles=raw_profiles, diff --git a/dbt/task/debug.py b/dbt/task/debug.py index a575791f6ff..dbaa2bc06e1 100644 --- a/dbt/task/debug.py +++ b/dbt/task/debug.py @@ -1,10 +1,18 @@ +# coding=utf-8 +import os +import platform import pprint +import sys from dbt.logger import GLOBAL_LOGGER as logger import dbt.clients.system import dbt.config import dbt.utils import dbt.exceptions +from dbt.adapters.factory import get_adapter +from dbt.version import get_installed_version +from dbt.config import Project, Profile +from dbt.clients.yaml_helper import load_yaml_text from dbt.task.base_task import BaseTask @@ -12,47 +20,259 @@ {open_cmd} {profiles_dir}""" +ONLY_PROFILE_MESSAGE = ''' +A `dbt_project.yml` file was not found in this directory. +Using the only profile `{}`. +'''.lstrip() + +MULTIPLE_PROFILE_MESSAGE = ''' +A `dbt_project.yml` file was not found in this directory. +dbt found the following profiles: +{} + +To debug one of these profiles, run: +dbt debug --profile [profile-name] +'''.lstrip() + +COULD_NOT_CONNECT_MESSAGE = ''' +dbt was unable to connect to the specified database. +The database returned the following error: + + >{} + +Check your database credentials and try again. For more information, visit: +https://docs.getdbt.com/docs/configure-your-profile +'''.lstrip() + + +MISSING_PROFILE_MESSAGE = ''' +dbt looked for a profiles.yml file in /Users/drew/.dbt/profiles.yml, but did +not find one. For more information on configuring your profile, consult the +documentation: + +https://docs.getdbt.com/docs/configure-your-profile +'''.lstrip() + +FILE_NOT_FOUND = 'file not found' + class DebugTask(BaseTask): + def __init__(self, args, config=None): + super(DebugTask, self).__init__(args, config) + self.profiles_dir = getattr(self.args, 'profiles_dir', + dbt.config.PROFILES_DIR) + self.profile_path = os.path.join(self.profiles_dir, 'profiles.yml') + self.project_path = os.path.join(os.getcwd(), 'dbt_project.yml') + self.cli_vars = dbt.utils.parse_cli_vars( + getattr(self.args, 'vars', '{}') + ) + + # set by _load_* + self.profile = None + self.profile_fail_details = '' + self.raw_profile_data = None + self.profile_name = None + self.project = None + self.project_fail_details = '' + self.messages = [] + + @property + def project_profile(self): + if self.project is None: + return None + return self.project.profile_name + def path_info(self): open_cmd = dbt.clients.system.open_dir_cmd() - profiles_dir = dbt.config.PROFILES_DIR message = PROFILE_DIR_MESSAGE.format( open_cmd=open_cmd, - profiles_dir=profiles_dir + profiles_dir=self.profiles_dir ) logger.info(message) - def diag(self): - # if we got here, a 'dbt_project.yml' does exist, but we have not tried - # to parse it. - project_profile = None - cli_vars = dbt.utils.parse_cli_vars(getattr(self.args, 'vars', '{}')) + def run(self): + version = get_installed_version().to_version_string(skip_matcher=True) + print('dbt version: {}'.format(version)) + print('python version: {}'.format(sys.version.split()[0])) + print('python path: {}'.format(sys.executable)) + print('os info: {}'.format(platform.platform())) + print('Using profiles.yml file at {}'.format(self.profile_path)) + print('') + self.test_configuration() + self.test_dependencies() + self.test_connection() + + for message in self.messages: + print(message) + print('') + + def _load_project(self): + if not os.path.exists(self.project_path): + self.project_fail_details = FILE_NOT_FOUND + return '✗ not found' try: - project = dbt.config.Project.from_current_directory(cli_vars) - project_profile = project.profile_name + self.project = Project.from_current_directory(self.cli_vars) except dbt.exceptions.DbtConfigError as exc: - project = 'ERROR loading project: {!s}'.format(exc) + self.project_fail_details = str(exc) + return '✗ invalid' + + return '✓ found and valid' + + def _profile_found(self): + if not self.raw_profile_data: + return '✗ not found' + if self.profile_name in self.raw_profile_data: + return '✓ found' + else: + return '✗ not found' + + def _target_found(self): + requirements = (self.raw_profile_data and self.profile_name and + self.target_name) + if not requirements: + return '✗ not found' + if self.profile_name not in self.raw_profile_data: + return '✗ not found' + profiles = self.raw_profile_data[self.profile_name]['outputs'] + if self.target_name not in profiles: + return '✗ not found' + return '✓ found' + + def _choose_profile_name(self): + assert self.project or self.project_fail_details, \ + '_load_project() required' + + project_profile = None + if self.project: + project_profile = self.project.profile_name + + args_profile = getattr(self.args, 'profile', None) - # log the profile we decided on as well, if it's available. try: - profile = dbt.config.Profile.from_args(self.args, project_profile, - cli_vars) - except dbt.exceptions.DbtConfigError as exc: - profile = 'ERROR loading profile: {!s}'.format(exc) + return Profile.pick_profile_name(args_profile, project_profile) + except dbt.exceptions.DbtConfigError: + pass + # try to guess - logger.info("args: {}".format(self.args)) - logger.info("") - logger.info("project:\n{!s}".format(project)) - logger.info("") - logger.info("profile:\n{!s}".format(profile)) + if self.raw_profile_data: + profiles = [k for k in self.raw_profile_data if k != 'config'] + if len(profiles) == 0: + self.messages.append('The profiles.yml has no profiles') + elif len(profiles) == 1: + self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0])) + return profiles[0] + else: + self.messages.append(MULTIPLE_PROFILE_MESSAGE.format( + '\n'.join(' - {}'.format(o) for o in profiles) + )) + return None - def run(self): + def _choose_target_name(self): + has_raw_profile = (self.raw_profile_data and self.profile_name and + self.profile_name in self.raw_profile_data) + if has_raw_profile: + raw_profile = self.raw_profile_data[self.profile_name] - if self.args.config_dir: - self.path_info() + target_name, _ = Profile.render_profile( + raw_profile, self.profile_name, + getattr(self.args, 'target', None), self.cli_vars + ) + return target_name + return None + + def _load_profile(self): + if not os.path.exists(self.profile_path): + self.profile_fail_details = FILE_NOT_FOUND + self.messages.append(MISSING_PROFILE_MESSAGE) + return '✗ not found' + + try: + raw_profile_data = load_yaml_text( + dbt.clients.system.load_file_contents(self.profile_path) + ) + except Exception: + pass # we'll report this when we try to load the profile for real else: - self.diag() + if isinstance(raw_profile_data, dict): + self.raw_profile_data = raw_profile_data + + self.profile_name = self._choose_profile_name() + self.target_name = self._choose_target_name() + try: + self.profile = Profile.from_args(self.args, self.profile_name, + self.cli_vars) + except dbt.exceptions.DbtConfigError as exc: + self.profile_fail_details = str(exc) + return '✗ invalid' + + return '✓ found and valid' + + def test_git(self): + try: + dbt.clients.system.run_cmd(os.getcwd(), ['git', '--help']) + except dbt.exceptions.ExecutableError as exc: + self.messages.append('Error from git --help: {!s}'.format(exc)) + return '✗ error' + return '✓ found' + + def test_dependencies(self): + print('Required dependencies:') + print(' - git [{}]'.format(self.test_git())) + print('') + + def test_configuration(self): + project_status = self._load_project() + profile_status = self._load_profile() + print('Configuration:') + print(' profiles.yml file [{}]'.format(profile_status)) + print(' dbt_project.yml file [{}]'.format(project_status)) + # skip profile stuff if we can't find a profile name + if self.profile_name is not None: + print(' profile: {} [{}]'.format(self.profile_name, + self._profile_found())) + print(' target: {} [{}]'.format(self.target_name, + self._target_found())) + print('') + self._log_project_fail() + self._log_profile_fail() + + def _log_project_fail(self): + if not self.project_fail_details: + return + if self.project_fail_details == FILE_NOT_FOUND: + return + print('Project loading failed for the following reason:') + print(self.project_fail_details) + print('') + + def _log_profile_fail(self): + if not self.profile_fail_details: + return + if self.profile_fail_details == FILE_NOT_FOUND: + return + if self.profile_name is None: + return # we expect an error (no profile provided) + print('Profile loading failed for the following reason:') + print(self.profile_fail_details) + print('') + + def _connection_result(self): + adapter = get_adapter(self.profile) + try: + adapter.execute('select 1 as id') + except Exception as exc: + self.messages.append(COULD_NOT_CONNECT_MESSAGE.format(str(exc))) + return '✗ error' + return '✓ connection ok' + + def test_connection(self): + if not self.profile: + return + print('Connection:') + for k, v in self.profile.credentials.connection_info(): + print(' {}: {}'.format(k, v)) + print(' Connection test: {}'.format(self._connection_result())) + print('')