dbt debug

dbt/config.py

@@ -543,7 +543,7 @@ def _credentials_from_profile(profile, profile_name, target_name):
  return credentials
 
  @staticmethod
- def _pick_profile_name(args_profile_name, project_profile_name=None):
+ def pick_profile_name(args_profile_name, project_profile_name=None):
  profile_name = project_profile_name
  if args_profile_name is not None:
  profile_name = args_profile_name
@@ -606,8 +606,8 @@ def from_credentials(cls, credentials, threads, profile_name, target_name,
  return profile
 
  @classmethod
- def _render_profile(cls, raw_profile, profile_name, target_override,
-  cli_vars):
+ def render_profile(cls, raw_profile, profile_name, target_override,
+ cli_vars):
  """This is a containment zone for the hateful way we're rendering
  profiles.
  """
@@ -664,7 +664,7 @@ def from_raw_profile_info(cls, raw_profile, profile_name, cli_vars,
  """
  # user_cfg is not rendered since it only contains booleans.
  # TODO: should it be, and the values coerced to bool?
- target_name, profile_data = cls._render_profile(
+ target_name, profile_data = cls.render_profile(
  raw_profile, profile_name, target_override, cli_vars
  )
 
@@ -749,8 +749,8 @@ def from_args(cls, args, project_profile_name=None, cli_vars=None):
  profiles_dir = getattr(args, 'profiles_dir', PROFILES_DIR)
  target_override = getattr(args, 'target', None)
  raw_profiles = read_profile(profiles_dir)
- profile_name = cls._pick_profile_name(args.profile,
-  project_profile_name)
+ profile_name = cls.pick_profile_name(args.profile,
+ project_profile_name)
 
  return cls.from_raw_profiles(
  raw_profiles=raw_profiles,

dbt/task/debug.py

@@ -1,58 +1,278 @@
+# coding=utf-8
+import os
+import platform
 import pprint
+import sys
 
 from dbt.logger import GLOBAL_LOGGER as logger
 import dbt.clients.system
 import dbt.config
 import dbt.utils
 import dbt.exceptions
+from dbt.adapters.factory import get_adapter
+from dbt.version import get_installed_version
+from dbt.config import Project, Profile
+from dbt.clients.yaml_helper import load_yaml_text
 
 from dbt.task.base_task import BaseTask
 
 PROFILE_DIR_MESSAGE = """To view your profiles.yml file, run:
 
 {open_cmd} {profiles_dir}"""
 
+ONLY_PROFILE_MESSAGE = '''
+A `dbt_project.yml` file was not found in this directory.
+Using the only profile `{}`.
+'''.lstrip()
+
+MULTIPLE_PROFILE_MESSAGE = '''
+A `dbt_project.yml` file was not found in this directory.
+dbt found the following profiles:
+{}
+
+To debug one of these profiles, run:
+dbt debug --profile [profile-name]
+'''.lstrip()
+
+COULD_NOT_CONNECT_MESSAGE = '''
+dbt was unable to connect to the specified database.
+The database returned the following error:
+
+ >{}
+
+Check your database credentials and try again. For more information, visit:
+https://docs.getdbt.com/docs/configure-your-profile
+'''.lstrip()
+
+
+MISSING_PROFILE_MESSAGE = '''
+dbt looked for a profiles.yml file in /Users/drew/.dbt/profiles.yml, but did
+not find one. For more information on configuring your profile, consult the
+documentation:
+
+https://docs.getdbt.com/docs/configure-your-profile
+'''.lstrip()
+
+FILE_NOT_FOUND = 'file not found'
+
 
 class DebugTask(BaseTask):
+ def __init__(self, args, config=None):
+ super(DebugTask, self).__init__(args, config)
+ self.profiles_dir = getattr(self.args, 'profiles_dir',
+ dbt.config.PROFILES_DIR)
+ self.profile_path = os.path.join(self.profiles_dir, 'profiles.yml')
+ self.project_path = os.path.join(os.getcwd(), 'dbt_project.yml')
+ self.cli_vars = dbt.utils.parse_cli_vars(
+ getattr(self.args, 'vars', '{}')
+ )
+
+ # set by _load_*
+ self.profile = None
+ self.profile_fail_details = ''
+ self.raw_profile_data = None
+ self.profile_name = None
+ self.project = None
+ self.project_fail_details = ''
+ self.messages = []
+
+ @property
+ def project_profile(self):
+ if self.project is None:
+ return None
+ return self.project.profile_name
+
  def path_info(self):
  open_cmd = dbt.clients.system.open_dir_cmd()
- profiles_dir = dbt.config.PROFILES_DIR
 
  message = PROFILE_DIR_MESSAGE.format(
  open_cmd=open_cmd,
- profiles_dir=profiles_dir
+ profiles_dir=self.profiles_dir
  )
 
  logger.info(message)
 
- def diag(self):
- # if we got here, a 'dbt_project.yml' does exist, but we have not tried
- # to parse it.
- project_profile = None
- cli_vars = dbt.utils.parse_cli_vars(getattr(self.args, 'vars', '{}'))
+ def run(self):
+ version = get_installed_version().to_version_string(skip_matcher=True)
+ print('dbt version: {}'.format(version))
+ print('python version: {}'.format(sys.version.split()[0]))
+ print('python path: {}'.format(sys.executable))
+ print('os info: {}'.format(platform.platform()))
+ print('Using profiles.yml file at {}'.format(self.profile_path))
+ print('')
+ self.test_configuration()
+ self.test_dependencies()
+ self.test_connection()
+
+ for message in self.messages:
+ print(message)
+ print('')
+
+ def _load_project(self):
+ if not os.path.exists(self.project_path):
+ self.project_fail_details = FILE_NOT_FOUND
+ return '✗ not found'
 
  try:
- project = dbt.config.Project.from_current_directory(cli_vars)
- project_profile = project.profile_name
+ self.project = Project.from_current_directory(self.cli_vars)
  except dbt.exceptions.DbtConfigError as exc:
- project = 'ERROR loading project: {!s}'.format(exc)
+ self.project_fail_details = str(exc)
+ return '✗ invalid'
+
+ return '✓ found and valid'
+
+ def _profile_found(self):
+ if not self.raw_profile_data:
+ return '✗ not found'
+ if self.profile_name in self.raw_profile_data:
+ return '✓ found'
+ else:
+ return '✗ not found'
+
+ def _target_found(self):
+ requirements = (self.raw_profile_data and self.profile_name and
+ self.target_name)
+ if not requirements:
+ return '✗ not found'
+ if self.profile_name not in self.raw_profile_data:
+ return '✗ not found'
+ profiles = self.raw_profile_data[self.profile_name]['outputs']
+ if self.target_name not in profiles:
+ return '✗ not found'
+ return '✓ found'
+
+ def _choose_profile_name(self):
+ assert self.project or self.project_fail_details, \
+ '_load_project() required'
+
+ project_profile = None
+ if self.project:
+ project_profile = self.project.profile_name
+
+ args_profile = getattr(self.args, 'profile', None)
 
- # log the profile we decided on as well, if it's available.
  try:
- profile = dbt.config.Profile.from_args(self.args, project_profile,
-  cli_vars)
- except dbt.exceptions.DbtConfigError as exc:
-  profile = 'ERROR loading profile: {!s}'.format(exc)
+ return Profile.pick_profile_name(args_profile, project_profile)
+ except dbt.exceptions.DbtConfigError:
+  pass
+ # try to guess
 
- logger.info("args: {}".format(self.args))
- logger.info("")
- logger.info("project:\n{!s}".format(project))
- logger.info("")
- logger.info("profile:\n{!s}".format(profile))
+ if self.raw_profile_data:
+ profiles = [k for k in self.raw_profile_data if k != 'config']
+ if len(profiles) == 0:
+ self.messages.append('The profiles.yml has no profiles')
+ elif len(profiles) == 1:
+ self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
+ return profiles[0]
+ else:
+ self.messages.append(MULTIPLE_PROFILE_MESSAGE.format(
+ '\n'.join(' - {}'.format(o) for o in profiles)
+ ))
+ return None
 
- def run(self):
+ def _choose_target_name(self):
+ has_raw_profile = (self.raw_profile_data and self.profile_name and
+ self.profile_name in self.raw_profile_data)
+ if has_raw_profile:
+ raw_profile = self.raw_profile_data[self.profile_name]
 
- if self.args.config_dir:
- self.path_info()
+ target_name, _ = Profile.render_profile(
+ raw_profile, self.profile_name,
+ getattr(self.args, 'target', None), self.cli_vars
+ )
+ return target_name
+ return None
+
+ def _load_profile(self):
+ if not os.path.exists(self.profile_path):
+ self.profile_fail_details = FILE_NOT_FOUND
+ self.messages.append(MISSING_PROFILE_MESSAGE)
+ return '✗ not found'
+
+ try:
+ raw_profile_data = load_yaml_text(
+ dbt.clients.system.load_file_contents(self.profile_path)
+ )
+ except Exception:
+ pass # we'll report this when we try to load the profile for real
  else:
- self.diag()
+ if isinstance(raw_profile_data, dict):
+ self.raw_profile_data = raw_profile_data
+
+ self.profile_name = self._choose_profile_name()
+ self.target_name = self._choose_target_name()
+ try:
+ self.profile = Profile.from_args(self.args, self.profile_name,
+ self.cli_vars)
+ except dbt.exceptions.DbtConfigError as exc:
+ self.profile_fail_details = str(exc)
+ return '✗ invalid'
+
+ return '✓ found and valid'
+
+ def test_git(self):
+ try:
+ dbt.clients.system.run_cmd(os.getcwd(), ['git', '--help'])
+ except dbt.exceptions.ExecutableError as exc:
+ self.messages.append('Error from git --help: {!s}'.format(exc))
+ return '✗ error'
+ return '✓ found'
+
+ def test_dependencies(self):
+ print('Required dependencies:')
+ print(' - git [{}]'.format(self.test_git()))
+ print('')
+
+ def test_configuration(self):
+ project_status = self._load_project()
+ profile_status = self._load_profile()
+ print('Configuration:')
+ print(' profiles.yml file [{}]'.format(profile_status))
+ print(' dbt_project.yml file [{}]'.format(project_status))
+ # skip profile stuff if we can't find a profile name
+ if self.profile_name is not None:
+ print(' profile: {} [{}]'.format(self.profile_name,
+ self._profile_found()))
+ print(' target: {} [{}]'.format(self.target_name,
+ self._target_found()))
+ print('')
+ self._log_project_fail()
+ self._log_profile_fail()
+
+ def _log_project_fail(self):
+ if not self.project_fail_details:
+ return
+ if self.project_fail_details == FILE_NOT_FOUND:
+ return
+ print('Project loading failed for the following reason:')
+ print(self.project_fail_details)
+ print('')
+
+ def _log_profile_fail(self):
+ if not self.profile_fail_details:
+ return
+ if self.profile_fail_details == FILE_NOT_FOUND:
+ return
+ if self.profile_name is None:
+ return # we expect an error (no profile provided)
+ print('Profile loading failed for the following reason:')
+ print(self.profile_fail_details)
+ print('')
+
+ def _connection_result(self):
+ adapter = get_adapter(self.profile)
+ try:
+ adapter.execute('select 1 as id')
+ except Exception as exc:
+ self.messages.append(COULD_NOT_CONNECT_MESSAGE.format(str(exc)))
+ return '✗ error'
+ return '✓ connection ok'
+
+ def test_connection(self):
+ if not self.profile:
+ return
+ print('Connection:')
+ for k, v in self.profile.credentials.connection_info():
+ print(' {}: {}'.format(k, v))
+ print(' Connection test: {}'.format(self._connection_result()))
+ print('')