Convert messages to struct logs (#6064)

* Initial structured logging changes

* remove "this" from core/dbt/events/functions.py

* CT-1047: Fix execution_time definitions to use float

* CT-1047: Revert unintended checking of changes to functions.py

* WIP

* first pass to resolve circular deps

* more circular dep resolution

* remove a bunch of duplication

* move message into log line

* update comments

* fix field that wen missing during rebase

* remove double import

* remove some comments and extra code

* fix pre-commit

* rework deprecations

* WIP converting messages

* WIP converting messages

* remove stray comment

* WIP more message conversion

* WIP more message conversion

* tweak the messages

* convert last message

* rename

* remove warn_or_raise as never used

* add fake calls to all new events

* fix some tests

* put back deprecation

* restore deprecation fully

* fix unit test

* fix log levels

* remove some skipped ids

* fix macro log function

* fix how messages are built to match expected outcome

* fix expected test message

* small fixes from reviews

* fix conflict resolution in UI

Co-authored-by: Gerda Shank <gerda@dbtlabs.com>
Co-authored-by: Peter Allen Webb <peter.webb@dbtlabs.com>

core/dbt/adapters/base/impl.py

@@ -41,13 +41,13 @@
 from dbt.contracts.graph.compiled import CompileResultNode, CompiledSeedNode
 from dbt.contracts.graph.manifest import Manifest, MacroManifest
 from dbt.contracts.graph.parsed import ParsedSeedNode
-from dbt.exceptions import warn_or_error
-from dbt.events.functions import fire_event
+from dbt.events.functions import fire_event, warn_or_error
 from dbt.events.types import (
     CacheMiss,
     ListRelations,
     CodeExecution,
     CodeExecutionStatus,
+    CatalogGenerationError,
 )
 from dbt.utils import filter_null_values, executor, cast_to_str
 
@@ -1327,7 +1327,7 @@ def catch_as_completed(
         elif isinstance(exc, KeyboardInterrupt) or not isinstance(exc, Exception):
             raise exc
         else:
-            warn_or_error(f"Encountered an error while generating catalog: {str(exc)}")
+            warn_or_error(CatalogGenerationError(exc=str(exc)))
             # exc is not None, derives from Exception, and isn't ctrl+c
             exceptions.append(exc)
     return merge_tables(tables), exceptions

core/dbt/config/runtime.py

@@ -8,7 +8,6 @@
     Dict,
     Iterable,
     Iterator,
-    List,
     Mapping,
     MutableSet,
     Optional,
@@ -30,10 +29,10 @@
     RuntimeException,
     raise_compiler_error,
     validator_error_message,
-    warn_or_error,
 )
+from dbt.events.functions import warn_or_error
+from dbt.events.types import UnusedResourceConfigPath
 from dbt.helper_types import DictDefaultEmptyStr, FQNPath, PathSet
-from dbt.ui import warning_tag
 
 from .profile import Profile
 from .project import Project, PartialProject
@@ -315,11 +314,11 @@ def get_resource_config_paths(self) -> Dict[str, PathSet]:
             "exposures": self._get_config_paths(self.exposures),
         }
 
-    def get_unused_resource_config_paths(
+    def warn_for_unused_resource_config_paths(
         self,
         resource_fqns: Mapping[str, PathSet],
         disabled: PathSet,
-    ) -> List[FQNPath]:
+    ) -> None:
         """Return a list of lists of strings, where each inner list of strings
         represents a type + FQN path of a resource configuration that is not
         used.
@@ -333,23 +332,13 @@ def get_unused_resource_config_paths(
 
             for config_path in config_paths:
                 if not _is_config_used(config_path, fqns):
-                    unused_resource_config_paths.append((resource_type,) + config_path)
-        return unused_resource_config_paths
+                    resource_path = ".".join(i for i in ((resource_type,) + config_path))
+                    unused_resource_config_paths.append(resource_path)
 
-    def warn_for_unused_resource_config_paths(
-        self,
-        resource_fqns: Mapping[str, PathSet],
-        disabled: PathSet,
-    ) -> None:
-        unused = self.get_unused_resource_config_paths(resource_fqns, disabled)
-        if len(unused) == 0:
+        if len(unused_resource_config_paths) == 0:
             return
 
-        msg = UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE.format(
-            len(unused), "\n".join("- {}".format(".".join(u)) for u in unused)
-        )
-
-        warn_or_error(msg, log_fmt=warning_tag("{}"))
+        warn_or_error(UnusedResourceConfigPath(unused_config_paths=unused_resource_config_paths))
 
     def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]:
         if self.dependencies is None:
@@ -626,14 +615,6 @@ def from_args(cls: Type[RuntimeConfig], args: Any) -> "RuntimeConfig":
         return cls.from_parts(project=project, profile=profile, args=args)
 
 
-UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE = """\
-Configuration paths exist in your dbt_project.yml file which do not \
-apply to any resources.
-There are {} unused configuration paths:
-{}
-"""
-
-
 def _is_config_used(path, fqns):
     if fqns:
         for fqn in fqns:

core/dbt/constants.py

@@ -1,3 +1,10 @@
 SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
 DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
 METADATA_ENV_PREFIX = "DBT_ENV_CUSTOM_ENV_"
+
+MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
+MAXIMUM_SEED_SIZE_NAME = "1MB"
+
+PIN_PACKAGE_URL = (
+    "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"
+)

core/dbt/context/providers.py

@@ -53,7 +53,6 @@
     raise_compiler_error,
     ref_invalid_args,
     metric_invalid_args,
-    ref_target_not_found,
     target_not_found,
     ref_bad_context,
     wrapped_exports,
@@ -476,10 +475,11 @@ def resolve(self, target_name: str, target_package: Optional[str] = None) -> Rel
         )
 
         if target_model is None or isinstance(target_model, Disabled):
-            ref_target_not_found(
-                self.model,
-                target_name,
-                target_package,
+            target_not_found(
+                node=self.model,
+                target_name=target_name,
+                target_kind="node",
+                target_package=target_package,
                 disabled=isinstance(target_model, Disabled),
             )
         self.validate(target_model, target_name, target_package)

core/dbt/contracts/files.py

@@ -1,18 +1,16 @@
 import hashlib
 import os
 from dataclasses import dataclass, field
+
 from mashumaro.types import SerializableType
 from typing import List, Optional, Union, Dict, Any
 
+from dbt.constants import MAXIMUM_SEED_SIZE
 from dbt.dataclass_schema import dbtClassMixin, StrEnum
 
 from .util import SourceKey
 
 
-MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
-MAXIMUM_SEED_SIZE_NAME = "1MB"
-
-
 class ParseFileType(StrEnum):
     Macro = "macro"
     Model = "model"

core/dbt/contracts/graph/parsed.py

@@ -18,7 +18,7 @@
 from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
 
 from dbt.clients.system import write_file
-from dbt.contracts.files import FileHash, MAXIMUM_SEED_SIZE_NAME
+from dbt.contracts.files import FileHash
 from dbt.contracts.graph.unparsed import (
     UnparsedNode,
     UnparsedDocumentation,
@@ -41,7 +41,13 @@
 )
 from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
 from dbt.events.proto_types import NodeInfo
-from dbt.exceptions import warn_or_error
+from dbt.events.functions import warn_or_error
+from dbt.events.types import (
+    SeedIncreased,
+    SeedExceedsLimitSamePath,
+    SeedExceedsLimitAndPathChanged,
+    SeedExceedsLimitChecksumChanged,
+)
 from dbt import flags
 from dbt.node_types import ModelLanguage, NodeType
 
@@ -375,30 +381,28 @@ def same_seeds(first: ParsedNode, second: ParsedNode) -> bool:
     if first.checksum.name == "path":
         msg: str
         if second.checksum.name != "path":
-            msg = (
-                f"Found a seed ({first.package_name}.{first.name}) "
-                f">{MAXIMUM_SEED_SIZE_NAME} in size. The previous file was "
-                f"<={MAXIMUM_SEED_SIZE_NAME}, so it has changed"
+            warn_or_error(
+                SeedIncreased(package_name=first.package_name, name=first.name), node=first
             )
         elif result:
-            msg = (
-                f"Found a seed ({first.package_name}.{first.name}) "
-                f">{MAXIMUM_SEED_SIZE_NAME} in size at the same path, dbt "
-                f"cannot tell if it has changed: assuming they are the same"
+            warn_or_error(
+                SeedExceedsLimitSamePath(package_name=first.package_name, name=first.name),
+                node=first,
             )
         elif not result:
-            msg = (
-                f"Found a seed ({first.package_name}.{first.name}) "
-                f">{MAXIMUM_SEED_SIZE_NAME} in size. The previous file was in "
-                f"a different location, assuming it has changed"
+            warn_or_error(
+                SeedExceedsLimitAndPathChanged(package_name=first.package_name, name=first.name),
+                node=first,
             )
         else:
-            msg = (
-                f"Found a seed ({first.package_name}.{first.name}) "
-                f">{MAXIMUM_SEED_SIZE_NAME} in size. The previous file had a "
-                f"checksum type of {second.checksum.name}, so it has changed"
+            warn_or_error(
+                SeedExceedsLimitChecksumChanged(
+                    package_name=first.package_name,
+                    name=first.name,
+                    checksum_name=second.checksum.name,
+                ),
+                node=first,
             )
-        warn_or_error(msg, node=first)
 
     return result
 

core/dbt/contracts/project.py

@@ -12,9 +12,7 @@
 from typing import Optional, List, Dict, Union, Any
 from mashumaro.types import SerializableType
 
-PIN_PACKAGE_URL = (
-    "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"  # noqa
-)
+
 DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True
 
 

core/dbt/deprecations.py

@@ -1,14 +1,14 @@
+import abc
 from typing import Optional, Set, List, Dict, ClassVar
 
 import dbt.exceptions
-from dbt import ui
 
 import dbt.tracking
 
 
 class DBTDeprecation:
     _name: ClassVar[Optional[str]] = None
-    _description: ClassVar[Optional[str]] = None
+    _event: ClassVar[Optional[str]] = None
 
     @property
     def name(self) -> str:
@@ -21,66 +21,50 @@ def track_deprecation_warn(self) -> None:
             dbt.tracking.track_deprecation_warn({"deprecation_name": self.name})
 
     @property
-    def description(self) -> str:
-        if self._description is not None:
-            return self._description
-        raise NotImplementedError("description not implemented for {}".format(self))
+    def event(self) -> abc.ABCMeta:
+        if self._event is not None:
+            module_path = dbt.events.types
+            class_name = self._event
+
+            try:
+                return getattr(module_path, class_name)
+            except AttributeError:
+                msg = f"Event Class `{class_name}` is not defined in `{module_path}`"
+                raise NameError(msg)
+        raise NotImplementedError("event not implemented for {}".format(self._event))
 
     def show(self, *args, **kwargs) -> None:
         if self.name not in active_deprecations:
-            desc = self.description.format(**kwargs)
-            msg = ui.line_wrap_message(desc, prefix="Deprecated functionality\n\n")
-            dbt.exceptions.warn_or_error(msg, log_fmt=ui.warning_tag("{}"))
+            event = self.event(**kwargs)
+            dbt.events.functions.warn_or_error(event)
             self.track_deprecation_warn()
             active_deprecations.add(self.name)
 
 
 class PackageRedirectDeprecation(DBTDeprecation):
     _name = "package-redirect"
-    _description = """\
-    The `{old_name}` package is deprecated in favor of `{new_name}`. Please update
-    your `packages.yml` configuration to use `{new_name}` instead.
-    """
+    _event = "PackageRedirectDeprecation"
 
 
 class PackageInstallPathDeprecation(DBTDeprecation):
     _name = "install-packages-path"
-    _description = """\
-    The default package install path has changed from `dbt_modules` to `dbt_packages`.
-    Please update `clean-targets` in `dbt_project.yml` and check `.gitignore` as well.
-    Or, set `packages-install-path: dbt_modules` if you'd like to keep the current value.
-    """
+    _event = "PackageInstallPathDeprecation"
 
 
-class ConfigPathDeprecation(DBTDeprecation):
-    _description = """\
-    The `{deprecated_path}` config has been renamed to `{exp_path}`.
-    Please update your `dbt_project.yml` configuration to reflect this change.
-    """
-
-
-class ConfigSourcePathDeprecation(ConfigPathDeprecation):
+class ConfigSourcePathDeprecation(DBTDeprecation):
     _name = "project-config-source-paths"
+    _event = "ConfigSourcePathDeprecation"
 
 
-class ConfigDataPathDeprecation(ConfigPathDeprecation):
+class ConfigDataPathDeprecation(DBTDeprecation):
     _name = "project-config-data-paths"
-
-
-_adapter_renamed_description = """\
-The adapter function `adapter.{old_name}` is deprecated and will be removed in
-a future release of dbt. Please use `adapter.{new_name}` instead.
-
-Documentation for {new_name} can be found here:
-
-    https://docs.getdbt.com/docs/adapter
-"""
+    _event = "ConfigDataPathDeprecation"
 
 
 def renamed_method(old_name: str, new_name: str):
     class AdapterDeprecationWarning(DBTDeprecation):
         _name = "adapter:{}".format(old_name)
-        _description = _adapter_renamed_description.format(old_name=old_name, new_name=new_name)
+        _event = "AdapterDeprecationWarning"
 
     dep = AdapterDeprecationWarning()
     deprecations_list.append(dep)
@@ -89,26 +73,12 @@ class AdapterDeprecationWarning(DBTDeprecation):
 
 class MetricAttributesRenamed(DBTDeprecation):
     _name = "metric-attr-renamed"
-    _description = """\
-dbt-core v1.3 renamed attributes for metrics:
-\n  'sql'              -> 'expression'
-\n  'type'             -> 'calculation_method'
-\n  'type: expression' -> 'calculation_method: derived'
-\nThe old metric parameter names will be fully deprecated in v1.4.
-\nPlease remove them from the metric definition of metric '{metric_name}'
-\nRelevant issue here: https://github.com/dbt-labs/dbt-core/issues/5849
-"""
+    _event = "MetricAttributesRenamed"
 
 
 class ExposureNameDeprecation(DBTDeprecation):
     _name = "exposure-name"
-    _description = """\
-    Starting in v1.3, the 'name' of an exposure should contain only letters, numbers, and underscores.
-    Exposures support a new property, 'label', which may contain spaces, capital letters, and special characters.
-    {exposure} does not follow this pattern.
-    Please update the 'name', and use the 'label' property for a human-friendly title.
-    This will raise an error in a future version of dbt-core.
-    """
+    _event = "ExposureNameDeprecation"
 
 
 def warn(name, *args, **kwargs):
@@ -125,12 +95,12 @@ def warn(name, *args, **kwargs):
 active_deprecations: Set[str] = set()
 
 deprecations_list: List[DBTDeprecation] = [
-    ExposureNameDeprecation(),
+    PackageRedirectDeprecation(),
+    PackageInstallPathDeprecation(),
     ConfigSourcePathDeprecation(),
     ConfigDataPathDeprecation(),
-    PackageInstallPathDeprecation(),
-    PackageRedirectDeprecation(),
     MetricAttributesRenamed(),
+    ExposureNameDeprecation(),
 ]
 
 deprecations: Dict[str, DBTDeprecation] = {d.name: d for d in deprecations_list}