Move manifest nodes to dbt/artifacts

.changes/unreleased/Under the Hood-20240207-122342.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move manifest nodes to artifacts
+time: 2024-02-07T12:23:42.909049-05:00
+custom:
+  Author: gshank
+  Issue: "9388"

core/dbt/artifacts/resources/__init__.py

@@ -1,7 +1,30 @@
-from dbt.artifacts.resources.base import BaseResource, GraphResource
+from dbt.artifacts.resources.base import BaseResource, GraphResource, FileHash, Docs
 
 # alias to latest resource definitions
-from dbt.artifacts.resources.v1.components import DependsOn, NodeVersion, RefArgs
+from dbt.artifacts.resources.v1.components import (
+    DependsOn,
+    NodeVersion,
+    RefArgs,
+    HasRelationMetadata,
+    ParsedNodeMandatory,
+    ParsedNode,
+    ColumnInfo,
+    CompiledNode,
+    InjectedCTE,
+    Contract,
+    DeferRelation,
+)
+from dbt.artifacts.resources.v1.manifest_nodes import (
+    AnalysisNode,
+    HookNode,
+    ModelNode,
+    SqlNode,
+    SeedNode,
+    SingularTestNode,
+    TestMetadata,
+    GenericTestNode,
+    SnapshotNode,
+)
 from dbt.artifacts.resources.v1.documentation import Documentation
 from dbt.artifacts.resources.v1.exposure import (
     Exposure,
@@ -10,7 +33,6 @@
     MaturityType,
 )
 from dbt.artifacts.resources.v1.macro import Macro, MacroDependsOn, MacroArgument
-from dbt.artifacts.resources.v1.docs import Docs
 from dbt.artifacts.resources.v1.group import Group
 from dbt.artifacts.resources.v1.metric import (
     ConstantPropertyInput,
@@ -50,3 +72,12 @@
     SemanticModel,
     SemanticModelConfig,
 )
+
+from dbt.artifacts.resources.v1.config import (
+    NodeAndTestConfig,
+    NodeConfig,
+    ModelConfig,
+    SeedConfig,
+    TestConfig,
+    SnapshotConfig,
+)

core/dbt/artifacts/resources/base.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
 from dbt_common.dataclass_schema import dbtClassMixin
 from dbt_common.contracts.util import Replaceable
-from typing import List
+from typing import List, Optional
+import hashlib
 
 from dbt.artifacts.resources.types import NodeType
 
@@ -19,3 +20,49 @@
 @dataclass
 class GraphResource(BaseResource):
     fqn: List[str]
+
+
+@dataclass
+class FileHash(dbtClassMixin):
+    name: str  # the hash type name
+    checksum: str  # the hashlib.hash_type().hexdigest() of the file contents
+
+    @classmethod
+    def empty(cls):
+        return FileHash(name="none", checksum="")
+
+    @classmethod
+    def path(cls, path: str):
+        return FileHash(name="path", checksum=path)
+
+    def __eq__(self, other):
+        if not isinstance(other, FileHash):
+            return NotImplemented
+
+        if self.name == "none" or self.name != other.name:
+            return False
+
+        return self.checksum == other.checksum
+
+    def compare(self, contents: str) -> bool:
+        """Compare the file contents with the given hash"""
+        if self.name == "none":
+            return False
+
+        return self.from_contents(contents, name=self.name) == self.checksum
+
+    @classmethod
+    def from_contents(cls, contents: str, name="sha256") -> "FileHash":
+        """Create a file hash from the given file contents. The hash is always
+        the utf-8 encoding of the contents given, because dbt only reads files
+        as utf-8.
+        """
+        data = contents.encode("utf-8")
+        checksum = hashlib.new(name, data).hexdigest()
+        return cls(name=name, checksum=checksum)
+
+
+@dataclass
+class Docs(dbtClassMixin):
+    show: bool = True
+    node_color: Optional[str] = None

core/dbt/artifacts/resources/types.py

@@ -54,3 +54,8 @@ class RunHookType(StrEnum):
 class ModelLanguage(StrEnum):
     python = "python"
     sql = "sql"
+
+
+class ModelHookType(StrEnum):
+    PreHook = "pre-hook"
+    PostHook = "post-hook"

core/dbt/artifacts/resources/v1/components.py

@@ -1,12 +1,29 @@
+import time
 from dataclasses import dataclass, field
-from dbt.artifacts.resources.v1.macro import MacroDependsOn
-from dbt_common.dataclass_schema import dbtClassMixin
-from typing import Dict, List, Optional, Union
+from dbt.artifacts.resources.base import GraphResource, FileHash, Docs
+from dbt.artifacts.resources.v1.config import NodeConfig
+from dbt_common.dataclass_schema import (
+    dbtClassMixin,
+    ExtensibleDbtClassMixin,
+)
+from dbt_common.contracts.config.properties import AdditionalPropertiesMixin
+from dbt_common.contracts.constraints import ColumnLevelConstraint
+from typing import Dict, List, Optional, Union, Any
 
 
 NodeVersion = Union[str, float]
 
 
+@dataclass
+class MacroDependsOn(dbtClassMixin):
+    macros: List[str] = field(default_factory=list)
+
+    # 'in' on lists is O(n) so this is O(n^2) for # of macros
+    def add_macro(self, value: str):
+        if value not in self.macros:
+            self.macros.append(value)
+
+
 @dataclass
 class DependsOn(MacroDependsOn):
     nodes: List[str] = field(default_factory=list)
@@ -35,3 +52,113 @@ def keyword_args(self) -> Dict[str, Optional[NodeVersion]]:
             return {"version": self.version}
         else:
             return {}
+
+
+@dataclass
+class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin):
+    """Used in all ManifestNodes and SourceDefinition"""
+
+    name: str
+    description: str = ""
+    meta: Dict[str, Any] = field(default_factory=dict)
+    data_type: Optional[str] = None
+    constraints: List[ColumnLevelConstraint] = field(default_factory=list)
+    quote: Optional[bool] = None
+    tags: List[str] = field(default_factory=list)
+    _extra: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class InjectedCTE(dbtClassMixin):
+    """Used in CompiledNodes as part of ephemeral model processing"""
+
+    id: str
+    sql: str
+
+
+@dataclass
+class Contract(dbtClassMixin):
+    enforced: bool = False
+    alias_types: bool = True
+    checksum: Optional[str] = None
+
+
+@dataclass
+class HasRelationMetadata(dbtClassMixin):
+    database: Optional[str]
+    schema: str
+
+    # Can't set database to None like it ought to be
+    # because it messes up the subclasses and default parameters
+    # so hack it here
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        if "database" not in data:
+            data["database"] = None
+        return data
+
+    @property
+    def quoting_dict(self) -> Dict[str, bool]:
+        if hasattr(self, "quoting"):
+            return self.quoting.to_dict(omit_none=True)
+        else:
+            return {}
+
+
+@dataclass
+class DeferRelation(HasRelationMetadata):
+    alias: str
+    relation_name: Optional[str]
+
+    @property
+    def identifier(self):
+        return self.alias
+
+
+@dataclass
+class ParsedNodeMandatory(GraphResource, HasRelationMetadata):
+    alias: str
+    checksum: FileHash
+    config: NodeConfig = field(default_factory=NodeConfig)
+
+    @property
+    def identifier(self):
+        return self.alias
+
+
+@dataclass
+class ParsedNode(ParsedNodeMandatory):
+    tags: List[str] = field(default_factory=list)
+    description: str = field(default="")
+    columns: Dict[str, ColumnInfo] = field(default_factory=dict)
+    meta: Dict[str, Any] = field(default_factory=dict)
+    group: Optional[str] = None
+    docs: Docs = field(default_factory=Docs)
+    patch_path: Optional[str] = None
+    build_path: Optional[str] = None
+    deferred: bool = False
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    created_at: float = field(default_factory=lambda: time.time())
+    config_call_dict: Dict[str, Any] = field(default_factory=dict)
+    relation_name: Optional[str] = None
+    raw_code: str = ""
+
+
+@dataclass
+class CompiledNode(ParsedNode):
+    """Contains attributes necessary for SQL files and nodes with refs, sources, etc,
+    so all ManifestNodes except SeedNode."""
+
+    language: str = "sql"
+    refs: List[RefArgs] = field(default_factory=list)
+    sources: List[List[str]] = field(default_factory=list)
+    metrics: List[List[str]] = field(default_factory=list)
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    compiled_path: Optional[str] = None
+    compiled: bool = False
+    compiled_code: Optional[str] = None
+    extra_ctes_injected: bool = False
+    extra_ctes: List[InjectedCTE] = field(default_factory=list)
+    _pre_injected_sql: Optional[str] = None
+    contract: Contract = field(default_factory=Contract)