dbt debug

dbt/config.py

@@ -543,7 +543,7 @@ def _credentials_from_profile(profile, profile_name, target_name):
         return credentials
 
     @staticmethod
-    def _pick_profile_name(args_profile_name, project_profile_name=None):
+    def pick_profile_name(args_profile_name, project_profile_name=None):
         profile_name = project_profile_name
         if args_profile_name is not None:
             profile_name = args_profile_name
@@ -606,8 +606,8 @@ def from_credentials(cls, credentials, threads, profile_name, target_name,
         return profile
 
     @classmethod
-    def _render_profile(cls, raw_profile, profile_name, target_override,
-                        cli_vars):
+    def render_profile(cls, raw_profile, profile_name, target_override,
+                       cli_vars):
         """This is a containment zone for the hateful way we're rendering
         profiles.
         """
@@ -664,7 +664,7 @@ def from_raw_profile_info(cls, raw_profile, profile_name, cli_vars,
         """
         # user_cfg is not rendered since it only contains booleans.
         # TODO: should it be, and the values coerced to bool?
-        target_name, profile_data = cls._render_profile(
+        target_name, profile_data = cls.render_profile(
             raw_profile, profile_name, target_override, cli_vars
         )
 
@@ -749,8 +749,8 @@ def from_args(cls, args, project_profile_name=None, cli_vars=None):
         profiles_dir = getattr(args, 'profiles_dir', PROFILES_DIR)
         target_override = getattr(args, 'target', None)
         raw_profiles = read_profile(profiles_dir)
-        profile_name = cls._pick_profile_name(args.profile,
-                                              project_profile_name)
+        profile_name = cls.pick_profile_name(args.profile,
+                                             project_profile_name)
 
         return cls.from_raw_profiles(
             raw_profiles=raw_profiles,

dbt/task/debug.py

@@ -1,58 +1,278 @@
+# coding=utf-8
+import os
+import platform
 import pprint
+import sys
 
 from dbt.logger import GLOBAL_LOGGER as logger
 import dbt.clients.system
 import dbt.config
 import dbt.utils
 import dbt.exceptions
+from dbt.adapters.factory import get_adapter
+from dbt.version import get_installed_version
+from dbt.config import Project, Profile
+from dbt.clients.yaml_helper import load_yaml_text
 
 from dbt.task.base_task import BaseTask
 
 PROFILE_DIR_MESSAGE = """To view your profiles.yml file, run:
 
 {open_cmd} {profiles_dir}"""
 
+ONLY_PROFILE_MESSAGE = '''
+A `dbt_project.yml` file was not found in this directory.
+Using the only profile `{}`.
+'''.lstrip()
+
+MULTIPLE_PROFILE_MESSAGE = '''
+A `dbt_project.yml` file was not found in this directory.
+dbt found the following profiles:
+{}
+
+To debug one of these profiles, run:
+dbt debug --profile [profile-name]
+'''.lstrip()
+
+COULD_NOT_CONNECT_MESSAGE = '''
+dbt was unable to connect to the specified database.
+The database returned the following error:
+
+  >{}
+
+Check your database credentials and try again. For more information, visit:
+https://docs.getdbt.com/docs/configure-your-profile
+'''.lstrip()
+
+
+MISSING_PROFILE_MESSAGE = '''
+dbt looked for a profiles.yml file in /Users/drew/.dbt/profiles.yml, but did
+not find one. For more information on configuring your profile, consult the
+documentation:
+
+https://docs.getdbt.com/docs/configure-your-profile
+'''.lstrip()
+
+FILE_NOT_FOUND = 'file not found'
+
 
 class DebugTask(BaseTask):
+    def __init__(self, args, config=None):
+        super(DebugTask, self).__init__(args, config)
+        self.profiles_dir = getattr(self.args, 'profiles_dir',
+                                    dbt.config.PROFILES_DIR)
+        self.profile_path = os.path.join(self.profiles_dir, 'profiles.yml')
+        self.project_path = os.path.join(os.getcwd(), 'dbt_project.yml')
+        self.cli_vars = dbt.utils.parse_cli_vars(
+            getattr(self.args, 'vars', '{}')
+        )
+
+        # set by _load_*
+        self.profile = None
+        self.profile_fail_details = ''
+        self.raw_profile_data = None
+        self.profile_name = None
+        self.project = None
+        self.project_fail_details = ''
+        self.messages = []
+
+    @property
+    def project_profile(self):
+        if self.project is None:
+            return None
+        return self.project.profile_name
+
     def path_info(self):
         open_cmd = dbt.clients.system.open_dir_cmd()
-        profiles_dir = dbt.config.PROFILES_DIR
 
         message = PROFILE_DIR_MESSAGE.format(
             open_cmd=open_cmd,
-            profiles_dir=profiles_dir
+            profiles_dir=self.profiles_dir
         )
 
         logger.info(message)
 
-    def diag(self):
-        # if we got here, a 'dbt_project.yml' does exist, but we have not tried
-        # to parse it.
-        project_profile = None
-        cli_vars = dbt.utils.parse_cli_vars(getattr(self.args, 'vars', '{}'))
+    def run(self):
+        version = get_installed_version().to_version_string(skip_matcher=True)
+        print('dbt version: {}'.format(version))
+        print('python version: {}'.format(sys.version.split()[0]))
+        print('python path: {}'.format(sys.executable))
+        print('os info: {}'.format(platform.platform()))
+        print('Using profiles.yml file at {}'.format(self.profile_path))
+        print('')
+        self.test_configuration()
+        self.test_dependencies()
+        self.test_connection()
+
+        for message in self.messages:
+            print(message)
+            print('')
+
+    def _load_project(self):
+        if not os.path.exists(self.project_path):
+            self.project_fail_details = FILE_NOT_FOUND
+            return '✗ not found'
 
         try:
-            project = dbt.config.Project.from_current_directory(cli_vars)
-            project_profile = project.profile_name
+            self.project = Project.from_current_directory(self.cli_vars)
         except dbt.exceptions.DbtConfigError as exc:
-            project = 'ERROR loading project: {!s}'.format(exc)
+            self.project_fail_details = str(exc)
+            return '✗ invalid'
+
+        return '✓ found and valid'
+
+    def _profile_found(self):
+        if not self.raw_profile_data:
+            return '✗ not found'
+        if self.profile_name in self.raw_profile_data:
+            return '✓ found'
+        else:
+            return '✗ not found'
+
+    def _target_found(self):
+        requirements = (self.raw_profile_data and self.profile_name and
+                        self.target_name)
+        if not requirements:
+            return '✗ not found'
+        if self.profile_name not in self.raw_profile_data:
+            return '✗ not found'
+        profiles = self.raw_profile_data[self.profile_name]['outputs']
+        if self.target_name not in profiles:
+            return '✗ not found'
+        return '✓ found'
+
+    def _choose_profile_name(self):
+        assert self.project or self.project_fail_details, \
+            '_load_project() required'
+
+        project_profile = None
+        if self.project:
+            project_profile = self.project.profile_name
+
+        args_profile = getattr(self.args, 'profile', None)
 
-        # log the profile we decided on as well, if it's available.
         try:
-            profile = dbt.config.Profile.from_args(self.args, project_profile,
-                                                   cli_vars)
-        except dbt.exceptions.DbtConfigError as exc:
-            profile = 'ERROR loading profile: {!s}'.format(exc)
+            return Profile.pick_profile_name(args_profile, project_profile)
+        except dbt.exceptions.DbtConfigError:
+            pass
+        # try to guess
 
-        logger.info("args: {}".format(self.args))
-        logger.info("")
-        logger.info("project:\n{!s}".format(project))
-        logger.info("")
-        logger.info("profile:\n{!s}".format(profile))
+        if self.raw_profile_data:
+            profiles = [k for k in self.raw_profile_data if k != 'config']
+            if len(profiles) == 0:
+                self.messages.append('The profiles.yml has no profiles')
+            elif len(profiles) == 1:
+                self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0]))
+                return profiles[0]
+            else:
+                self.messages.append(MULTIPLE_PROFILE_MESSAGE.format(
+                    '\n'.join(' - {}'.format(o) for o in profiles)
+                ))
+        return None
 
-    def run(self):
+    def _choose_target_name(self):
+        has_raw_profile = (self.raw_profile_data and self.profile_name and
+                           self.profile_name in self.raw_profile_data)
+        if has_raw_profile:
+            raw_profile = self.raw_profile_data[self.profile_name]
 
-        if self.args.config_dir:
-            self.path_info()
+            target_name, _ = Profile.render_profile(
+                raw_profile, self.profile_name,
+                getattr(self.args, 'target', None), self.cli_vars
+            )
+            return target_name
+        return None
+
+    def _load_profile(self):
+        if not os.path.exists(self.profile_path):
+            self.profile_fail_details = FILE_NOT_FOUND
+            self.messages.append(MISSING_PROFILE_MESSAGE)
+            return '✗ not found'
+
+        try:
+            raw_profile_data = load_yaml_text(
+                dbt.clients.system.load_file_contents(self.profile_path)
+            )
+        except Exception:
+            pass  # we'll report this when we try to load the profile for real
         else:
-            self.diag()
+            if isinstance(raw_profile_data, dict):
+                self.raw_profile_data = raw_profile_data
+
+        self.profile_name = self._choose_profile_name()
+        self.target_name = self._choose_target_name()
+        try:
+            self.profile = Profile.from_args(self.args, self.profile_name,
+                                             self.cli_vars)
+        except dbt.exceptions.DbtConfigError as exc:
+            self.profile_fail_details = str(exc)
+            return '✗ invalid'
+
+        return '✓ found and valid'
+
+    def test_git(self):
+        try:
+            dbt.clients.system.run_cmd(os.getcwd(), ['git', '--help'])
+        except dbt.exceptions.ExecutableError as exc:
+            self.messages.append('Error from git --help: {!s}'.format(exc))
+            return '✗ error'
+        return '✓ found'
+
+    def test_dependencies(self):
+        print('Required dependencies:')
+        print(' - git [{}]'.format(self.test_git()))
+        print('')
+
+    def test_configuration(self):
+        project_status = self._load_project()
+        profile_status = self._load_profile()
+        print('Configuration:')
+        print('  profiles.yml file [{}]'.format(profile_status))
+        print('  dbt_project.yml file [{}]'.format(project_status))
+        # skip profile stuff if we can't find a profile name
+        if self.profile_name is not None:
+            print('  profile: {} [{}]'.format(self.profile_name,
+                                              self._profile_found()))
+            print('  target: {} [{}]'.format(self.target_name,
+                                             self._target_found()))
+        print('')
+        self._log_project_fail()
+        self._log_profile_fail()
+
+    def _log_project_fail(self):
+        if not self.project_fail_details:
+            return
+        if self.project_fail_details == FILE_NOT_FOUND:
+            return
+        print('Project loading failed for the following reason:')
+        print(self.project_fail_details)
+        print('')
+
+    def _log_profile_fail(self):
+        if not self.profile_fail_details:
+            return
+        if self.profile_fail_details == FILE_NOT_FOUND:
+            return
+        if self.profile_name is None:
+            return  # we expect an error (no profile provided)
+        print('Profile loading failed for the following reason:')
+        print(self.profile_fail_details)
+        print('')
+
+    def _connection_result(self):
+        adapter = get_adapter(self.profile)
+        try:
+            adapter.execute('select 1 as id')
+        except Exception as exc:
+            self.messages.append(COULD_NOT_CONNECT_MESSAGE.format(str(exc)))
+            return '✗ error'
+        return '✓ connection ok'
+
+    def test_connection(self):
+        if not self.profile:
+            return
+        print('Connection:')
+        for k, v in self.profile.credentials.connection_info():
+            print('  {}: {}'.format(k, v))
+        print('  Connection test: {}'.format(self._connection_result()))
+        print('')