Move WritableManifest to dbt/artifacts (#9377)

.changes/unreleased/Under the Hood-20240123-142256.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move WritableManifest + Documentation to dbt/artifacts
+time: 2024-01-23T14:22:56.488252-05:00
+custom:
+  Author: michelleark
+  Issue: 9378 9379

core/dbt/artifacts/resources/__init__.py

@@ -0,0 +1,4 @@
+from dbt.artifacts.resources.base import BaseArtifactNode
+
+# alias to latest resource definitions
+from dbt.artifacts.resources.v1.documentation import Documentation

core/dbt/artifacts/resources/base.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.contracts.util import Replaceable
+
+from dbt.artifacts.resources.types import NodeType
+
+
+@dataclass
+class BaseArtifactNode(dbtClassMixin, Replaceable):
+    name: str
+    resource_type: NodeType
+    package_name: str
+    path: str
+    original_file_path: str
+    unique_id: str

core/dbt/artifacts/resources/types.py

@@ -0,0 +1,54 @@
+from dbt_common.dataclass_schema import StrEnum
+
+
+class AccessType(StrEnum):
+    Private = "private"
+    Protected = "protected"
+    Public = "public"
+
+    @classmethod
+    def is_valid(cls, item):
+        try:
+            cls(item)
+        except ValueError:
+            return False
+        return True
+
+
+class NodeType(StrEnum):
+    Model = "model"
+    Analysis = "analysis"
+    Test = "test"
+    Snapshot = "snapshot"
+    Operation = "operation"
+    Seed = "seed"
+    # TODO: rm?
+    RPCCall = "rpc"
+    SqlOperation = "sql_operation"
+    Documentation = "doc"
+    Source = "source"
+    Macro = "macro"
+    Exposure = "exposure"
+    Metric = "metric"
+    Group = "group"
+    SavedQuery = "saved_query"
+    SemanticModel = "semantic_model"
+    Unit = "unit_test"
+    Fixture = "fixture"
+
+    def pluralize(self) -> str:
+        if self is self.Analysis:
+            return "analyses"
+        elif self is self.SavedQuery:
+            return "saved_queries"
+        return f"{self}s"
+
+
+class RunHookType(StrEnum):
+    Start = "on-run-start"
+    End = "on-run-end"
+
+
+class ModelLanguage(StrEnum):
+    python = "python"
+    sql = "sql"

core/dbt/artifacts/resources/v1/documentation.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass
+from typing import Literal
+
+from dbt.artifacts.resources.base import BaseArtifactNode
+from dbt.artifacts.resources.types import NodeType
+
+
+@dataclass
+class Documentation(BaseArtifactNode):
+    resource_type: Literal[NodeType.Documentation]
+    block_contents: str

core/dbt/artifacts/schemas/catalog/__init__.py

@@ -0,0 +1,2 @@
+# alias to latest
+from dbt.artifacts.schemas.catalog.v1.catalog import *  # noqa

core/dbt/artifacts/catalog.py → core/dbt/artifacts/schemas/catalog/v1/catalog.py

@@ -5,7 +5,7 @@
 from dbt_common.dataclass_schema import dbtClassMixin
 from dbt_common.utils.formatting import lowercase
 from dbt_common.contracts.util import Replaceable
-from dbt.artifacts.base import ArtifactMixin, BaseArtifactMetadata, schema_version
+from dbt.artifacts.schemas.base import ArtifactMixin, BaseArtifactMetadata, schema_version
 
 Primitive = Union[bool, str, float, None]
 PrimitiveDict = Dict[str, Primitive]

core/dbt/artifacts/schemas/freshness/__init__.py

@@ -0,0 +1 @@
+from dbt.artifacts.schemas.freshness.v3.freshness import *  # noqa

core/dbt/artifacts/freshness.py → core/dbt/artifacts/schemas/freshness/v3/freshness.py

@@ -2,8 +2,13 @@
 from typing import Dict, Any, Sequence, List, Union, Optional
 from datetime import datetime
 
-from dbt.artifacts.results import ExecutionResult, FreshnessStatus, NodeResult, TimingInfo
-from dbt.artifacts.base import ArtifactMixin, VersionedSchema, schema_version, BaseArtifactMetadata
+from dbt.artifacts.schemas.results import ExecutionResult, FreshnessStatus, NodeResult, TimingInfo
+from dbt.artifacts.schemas.base import (
+    ArtifactMixin,
+    VersionedSchema,
+    schema_version,
+    BaseArtifactMetadata,
+)
 from dbt_common.dataclass_schema import dbtClassMixin, StrEnum
 from dbt_common.exceptions import DbtInternalError
 

core/dbt/artifacts/schemas/manifest/__init__.py

@@ -0,0 +1,2 @@
+# alias to latest
+from dbt.artifacts.schemas.manifest.v12.manifest import *  # noqa

core/dbt/artifacts/schemas/manifest/v12/manifest.py

@@ -0,0 +1,180 @@
+from dataclasses import dataclass, field
+from typing import Mapping, Iterable, Tuple, Optional, Dict, List, Any
+from uuid import UUID
+
+from dbt.artifacts.schemas.base import (
+    BaseArtifactMetadata,
+    ArtifactMixin,
+    schema_version,
+    get_artifact_schema_version,
+)
+from dbt.artifacts.schemas.upgrades import upgrade_manifest_json
+from dbt.artifacts.resources import Documentation
+
+# TODO: remove usage of dbt modules other than dbt.artifacts
+from dbt import tracking
+from dbt.flags import get_flags
+from dbt.contracts.graph.nodes import (
+    Exposure,
+    GraphMemberNode,
+    Group,
+    Macro,
+    ManifestNode,
+    Metric,
+    SavedQuery,
+    SemanticModel,
+    SourceDefinition,
+    UnitTestDefinition,
+)
+
+
+NodeEdgeMap = Dict[str, List[str]]
+UniqueID = str
+
+
+@dataclass
+class ManifestMetadata(BaseArtifactMetadata):
+    """Metadata for the manifest."""
+
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(WritableManifest.dbt_schema_version)
+    )
+    project_name: Optional[str] = field(
+        default=None,
+        metadata={
+            "description": "Name of the root project",
+        },
+    )
+    project_id: Optional[str] = field(
+        default=None,
+        metadata={
+            "description": "A unique identifier for the project, hashed from the project name",
+        },
+    )
+    user_id: Optional[UUID] = field(
+        default=None,
+        metadata={
+            "description": "A unique identifier for the user",
+        },
+    )
+    send_anonymous_usage_stats: Optional[bool] = field(
+        default=None,
+        metadata=dict(
+            description=("Whether dbt is configured to send anonymous usage statistics")
+        ),
+    )
+    adapter_type: Optional[str] = field(
+        default=None,
+        metadata=dict(description="The type name of the adapter"),
+    )
+
+    def __post_init__(self):
+        if tracking.active_user is None:
+            return
+
+        if self.user_id is None:
+            self.user_id = tracking.active_user.id
+
+        if self.send_anonymous_usage_stats is None:
+            self.send_anonymous_usage_stats = get_flags().SEND_ANONYMOUS_USAGE_STATS
+
+    @classmethod
+    def default(cls):
+        return cls(
+            dbt_schema_version=str(WritableManifest.dbt_schema_version),
+        )
+
+
+@dataclass
+@schema_version("manifest", 12)
+class WritableManifest(ArtifactMixin):
+    nodes: Mapping[UniqueID, ManifestNode] = field(
+        metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
+    )
+    sources: Mapping[UniqueID, SourceDefinition] = field(
+        metadata=dict(description=("The sources defined in the dbt project and its dependencies"))
+    )
+    macros: Mapping[UniqueID, Macro] = field(
+        metadata=dict(description=("The macros defined in the dbt project and its dependencies"))
+    )
+    docs: Mapping[UniqueID, Documentation] = field(
+        metadata=dict(description=("The docs defined in the dbt project and its dependencies"))
+    )
+    exposures: Mapping[UniqueID, Exposure] = field(
+        metadata=dict(
+            description=("The exposures defined in the dbt project and its dependencies")
+        )
+    )
+    metrics: Mapping[UniqueID, Metric] = field(
+        metadata=dict(description=("The metrics defined in the dbt project and its dependencies"))
+    )
+    groups: Mapping[UniqueID, Group] = field(
+        metadata=dict(description=("The groups defined in the dbt project"))
+    )
+    selectors: Mapping[UniqueID, Any] = field(
+        metadata=dict(description=("The selectors defined in selectors.yml"))
+    )
+    disabled: Optional[Mapping[UniqueID, List[GraphMemberNode]]] = field(
+        metadata=dict(description="A mapping of the disabled nodes in the target")
+    )
+    parent_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from child nodes to their dependencies",
+        )
+    )
+    child_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from parent nodes to their dependents",
+        )
+    )
+    group_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from group names to their nodes",
+        )
+    )
+    saved_queries: Mapping[UniqueID, SavedQuery] = field(
+        metadata=dict(description=("The saved queries defined in the dbt project"))
+    )
+    semantic_models: Mapping[UniqueID, SemanticModel] = field(
+        metadata=dict(description=("The semantic models defined in the dbt project"))
+    )
+    metadata: ManifestMetadata = field(
+        metadata=dict(
+            description="Metadata about the manifest",
+        )
+    )
+    unit_tests: Mapping[UniqueID, UnitTestDefinition] = field(
+        metadata=dict(
+            description="The unit tests defined in the project",
+        )
+    )
+
+    @classmethod
+    def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
+        return [
+            ("manifest", 4),
+            ("manifest", 5),
+            ("manifest", 6),
+            ("manifest", 7),
+            ("manifest", 8),
+            ("manifest", 9),
+            ("manifest", 10),
+            ("manifest", 11),
+        ]
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This overrides the "upgrade_schema_version" call in VersionedSchema (via
+        ArtifactMixin) to modify the dictionary passed in from earlier versions of the manifest."""
+        manifest_schema_version = get_artifact_schema_version(data)
+        if manifest_schema_version <= 10:
+            data = upgrade_manifest_json(data, manifest_schema_version)
+        return cls.from_dict(data)
+
+    def __post_serialize__(self, dct):
+        for unique_id, node in dct["nodes"].items():
+            if "config_call_dict" in node:
+                del node["config_call_dict"]
+            if "defer_relation" in node:
+                del node["defer_relation"]
+        return dct

core/dbt/artifacts/schemas/run/__init__.py

@@ -0,0 +1,2 @@
+# alias to latest
+from dbt.artifacts.schemas.run.v5.run import *  # noqa

core/dbt/artifacts/run.py → core/dbt/artifacts/schemas/run/v5/run.py

@@ -6,13 +6,13 @@
 
 
 from dbt.contracts.graph.nodes import CompiledNode
-from dbt.artifacts.base import (
+from dbt.artifacts.schemas.base import (
     BaseArtifactMetadata,
     ArtifactMixin,
     schema_version,
     get_artifact_schema_version,
 )
-from dbt.artifacts.results import (
+from dbt.artifacts.schemas.results import (
     BaseResult,
     NodeResult,
     RunStatus,

core/dbt/artifacts/schemas/upgrades/__init__.py

@@ -0,0 +1 @@
+from dbt.artifacts.schemas.upgrades.upgrade_manifest import upgrade_manifest_json

core/dbt/cli/main.py

@@ -17,8 +17,8 @@
     DbtUsageException,
 )
 from dbt.contracts.graph.manifest import Manifest
-from dbt.artifacts.catalog import CatalogArtifact
-from dbt.artifacts.run import RunExecutionResult
+from dbt.artifacts.schemas.catalog import CatalogArtifact
+from dbt.artifacts.schemas.run import RunExecutionResult
 from dbt_common.events.base_types import EventMsg
 from dbt.task.build import BuildTask
 from dbt.task.clean import CleanTask