From ad723a6db897d9779f817da522758c05dce65730 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 24 Jan 2024 11:57:24 -0500 Subject: [PATCH] Move WritableManifest to dbt/artifacts (#9377) --- .../Under the Hood-20240123-142256.yaml | 6 + core/dbt/artifacts/__init__.py | 0 core/dbt/artifacts/resources/__init__.py | 4 + core/dbt/artifacts/resources/base.py | 15 ++ core/dbt/artifacts/resources/types.py | 54 ++++++ .../artifacts/resources/v1/documentation.py | 11 ++ core/dbt/artifacts/schemas/__init__.py | 0 core/dbt/artifacts/{ => schemas}/base.py | 0 .../dbt/artifacts/schemas/catalog/__init__.py | 2 + .../artifacts/schemas/catalog/v1/__init__.py | 0 .../{ => schemas/catalog/v1}/catalog.py | 2 +- .../artifacts/schemas/freshness/__init__.py | 1 + .../schemas/freshness/v3/__init__.py | 0 .../{ => schemas/freshness/v3}/freshness.py | 9 +- .../artifacts/schemas/manifest/__init__.py | 2 + .../schemas/manifest/v12/__init__.py | 0 .../schemas/manifest/v12/manifest.py | 180 ++++++++++++++++++ core/dbt/artifacts/{ => schemas}/results.py | 0 core/dbt/artifacts/schemas/run/__init__.py | 2 + core/dbt/artifacts/schemas/run/v5/__init__.py | 0 .../dbt/artifacts/{ => schemas/run/v5}/run.py | 4 +- .../artifacts/schemas/upgrades/__init__.py | 1 + .../schemas/upgrades/upgrade_manifest.py} | 0 core/dbt/cli/main.py | 4 +- core/dbt/contracts/graph/manifest.py | 173 +---------------- core/dbt/contracts/graph/nodes.py | 30 ++- core/dbt/contracts/graph/unparsed.py | 4 +- core/dbt/contracts/results.py | 10 +- core/dbt/contracts/sql.py | 6 +- core/dbt/contracts/state.py | 4 +- core/dbt/exceptions.py | 4 +- core/dbt/node_types.py | 106 +++-------- core/dbt/parser/manifest.py | 2 +- core/dbt/plugins/contracts.py | 4 +- core/dbt/task/base.py | 6 +- core/dbt/task/build.py | 4 +- core/dbt/task/clone.py | 10 +- core/dbt/task/compile.py | 6 +- core/dbt/task/debug.py | 2 +- core/dbt/task/docs/generate.py | 8 +- core/dbt/task/freshness.py | 2 +- core/dbt/task/printer.py | 2 +- core/dbt/task/retry.py | 2 +- core/dbt/task/run.py | 4 +- core/dbt/task/run_operation.py | 4 +- core/dbt/task/runnable.py | 4 +- core/dbt/task/seed.py | 2 +- core/dbt/task/show.py | 2 +- core/dbt/task/snapshot.py | 2 +- core/dbt/task/test.py | 6 +- scripts/collect-artifact-schema.py | 8 +- .../tests/adapter/basic/test_incremental.py | 2 +- .../tests/adapter/catalog/relation_types.py | 2 +- .../incremental/test_incremental_unique_id.py | 2 +- .../store_test_failures_tests/basic.py | 2 +- tests/functional/artifacts/test_artifacts.py | 4 +- .../artifacts/test_previous_version_state.py | 4 +- tests/unit/test_contracts_graph_unparsed.py | 2 +- tests/unit/test_events.py | 4 +- 59 files changed, 400 insertions(+), 336 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240123-142256.yaml create mode 100644 core/dbt/artifacts/__init__.py create mode 100644 core/dbt/artifacts/resources/__init__.py create mode 100644 core/dbt/artifacts/resources/base.py create mode 100644 core/dbt/artifacts/resources/types.py create mode 100644 core/dbt/artifacts/resources/v1/documentation.py create mode 100644 core/dbt/artifacts/schemas/__init__.py rename core/dbt/artifacts/{ => schemas}/base.py (100%) create mode 100644 core/dbt/artifacts/schemas/catalog/__init__.py create mode 100644 core/dbt/artifacts/schemas/catalog/v1/__init__.py rename core/dbt/artifacts/{ => schemas/catalog/v1}/catalog.py (96%) create mode 100644 core/dbt/artifacts/schemas/freshness/__init__.py create mode 100644 core/dbt/artifacts/schemas/freshness/v3/__init__.py rename core/dbt/artifacts/{ => schemas/freshness/v3}/freshness.py (94%) create mode 100644 core/dbt/artifacts/schemas/manifest/__init__.py create mode 100644 core/dbt/artifacts/schemas/manifest/v12/__init__.py create mode 100644 core/dbt/artifacts/schemas/manifest/v12/manifest.py rename core/dbt/artifacts/{ => schemas}/results.py (100%) create mode 100644 core/dbt/artifacts/schemas/run/__init__.py create mode 100644 core/dbt/artifacts/schemas/run/v5/__init__.py rename core/dbt/artifacts/{ => schemas/run/v5}/run.py (98%) create mode 100644 core/dbt/artifacts/schemas/upgrades/__init__.py rename core/dbt/{contracts/graph/manifest_upgrade.py => artifacts/schemas/upgrades/upgrade_manifest.py} (100%) diff --git a/.changes/unreleased/Under the Hood-20240123-142256.yaml b/.changes/unreleased/Under the Hood-20240123-142256.yaml new file mode 100644 index 00000000000..fd3ea99ef62 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240123-142256.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move WritableManifest + Documentation to dbt/artifacts +time: 2024-01-23T14:22:56.488252-05:00 +custom: + Author: michelleark + Issue: 9378 9379 diff --git a/core/dbt/artifacts/__init__.py b/core/dbt/artifacts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/artifacts/resources/__init__.py b/core/dbt/artifacts/resources/__init__.py new file mode 100644 index 00000000000..e7c97147b06 --- /dev/null +++ b/core/dbt/artifacts/resources/__init__.py @@ -0,0 +1,4 @@ +from dbt.artifacts.resources.base import BaseArtifactNode + +# alias to latest resource definitions +from dbt.artifacts.resources.v1.documentation import Documentation diff --git a/core/dbt/artifacts/resources/base.py b/core/dbt/artifacts/resources/base.py new file mode 100644 index 00000000000..72ce677e1ce --- /dev/null +++ b/core/dbt/artifacts/resources/base.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from dbt_common.dataclass_schema import dbtClassMixin +from dbt_common.contracts.util import Replaceable + +from dbt.artifacts.resources.types import NodeType + + +@dataclass +class BaseArtifactNode(dbtClassMixin, Replaceable): + name: str + resource_type: NodeType + package_name: str + path: str + original_file_path: str + unique_id: str diff --git a/core/dbt/artifacts/resources/types.py b/core/dbt/artifacts/resources/types.py new file mode 100644 index 00000000000..447142881a8 --- /dev/null +++ b/core/dbt/artifacts/resources/types.py @@ -0,0 +1,54 @@ +from dbt_common.dataclass_schema import StrEnum + + +class AccessType(StrEnum): + Private = "private" + Protected = "protected" + Public = "public" + + @classmethod + def is_valid(cls, item): + try: + cls(item) + except ValueError: + return False + return True + + +class NodeType(StrEnum): + Model = "model" + Analysis = "analysis" + Test = "test" + Snapshot = "snapshot" + Operation = "operation" + Seed = "seed" + # TODO: rm? + RPCCall = "rpc" + SqlOperation = "sql_operation" + Documentation = "doc" + Source = "source" + Macro = "macro" + Exposure = "exposure" + Metric = "metric" + Group = "group" + SavedQuery = "saved_query" + SemanticModel = "semantic_model" + Unit = "unit_test" + Fixture = "fixture" + + def pluralize(self) -> str: + if self is self.Analysis: + return "analyses" + elif self is self.SavedQuery: + return "saved_queries" + return f"{self}s" + + +class RunHookType(StrEnum): + Start = "on-run-start" + End = "on-run-end" + + +class ModelLanguage(StrEnum): + python = "python" + sql = "sql" diff --git a/core/dbt/artifacts/resources/v1/documentation.py b/core/dbt/artifacts/resources/v1/documentation.py new file mode 100644 index 00000000000..b62846512d0 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/documentation.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import Literal + +from dbt.artifacts.resources.base import BaseArtifactNode +from dbt.artifacts.resources.types import NodeType + + +@dataclass +class Documentation(BaseArtifactNode): + resource_type: Literal[NodeType.Documentation] + block_contents: str diff --git a/core/dbt/artifacts/schemas/__init__.py b/core/dbt/artifacts/schemas/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/artifacts/base.py b/core/dbt/artifacts/schemas/base.py similarity index 100% rename from core/dbt/artifacts/base.py rename to core/dbt/artifacts/schemas/base.py diff --git a/core/dbt/artifacts/schemas/catalog/__init__.py b/core/dbt/artifacts/schemas/catalog/__init__.py new file mode 100644 index 00000000000..1123f801adf --- /dev/null +++ b/core/dbt/artifacts/schemas/catalog/__init__.py @@ -0,0 +1,2 @@ +# alias to latest +from dbt.artifacts.schemas.catalog.v1.catalog import * # noqa diff --git a/core/dbt/artifacts/schemas/catalog/v1/__init__.py b/core/dbt/artifacts/schemas/catalog/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/artifacts/catalog.py b/core/dbt/artifacts/schemas/catalog/v1/catalog.py similarity index 96% rename from core/dbt/artifacts/catalog.py rename to core/dbt/artifacts/schemas/catalog/v1/catalog.py index b5307576e14..76a86e237fb 100644 --- a/core/dbt/artifacts/catalog.py +++ b/core/dbt/artifacts/schemas/catalog/v1/catalog.py @@ -5,7 +5,7 @@ from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.utils.formatting import lowercase from dbt_common.contracts.util import Replaceable -from dbt.artifacts.base import ArtifactMixin, BaseArtifactMetadata, schema_version +from dbt.artifacts.schemas.base import ArtifactMixin, BaseArtifactMetadata, schema_version Primitive = Union[bool, str, float, None] PrimitiveDict = Dict[str, Primitive] diff --git a/core/dbt/artifacts/schemas/freshness/__init__.py b/core/dbt/artifacts/schemas/freshness/__init__.py new file mode 100644 index 00000000000..4d04d4acf97 --- /dev/null +++ b/core/dbt/artifacts/schemas/freshness/__init__.py @@ -0,0 +1 @@ +from dbt.artifacts.schemas.freshness.v3.freshness import * # noqa diff --git a/core/dbt/artifacts/schemas/freshness/v3/__init__.py b/core/dbt/artifacts/schemas/freshness/v3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/artifacts/freshness.py b/core/dbt/artifacts/schemas/freshness/v3/freshness.py similarity index 94% rename from core/dbt/artifacts/freshness.py rename to core/dbt/artifacts/schemas/freshness/v3/freshness.py index b77749bd8ac..9e5cbc0cacc 100644 --- a/core/dbt/artifacts/freshness.py +++ b/core/dbt/artifacts/schemas/freshness/v3/freshness.py @@ -2,8 +2,13 @@ from typing import Dict, Any, Sequence, List, Union, Optional from datetime import datetime -from dbt.artifacts.results import ExecutionResult, FreshnessStatus, NodeResult, TimingInfo -from dbt.artifacts.base import ArtifactMixin, VersionedSchema, schema_version, BaseArtifactMetadata +from dbt.artifacts.schemas.results import ExecutionResult, FreshnessStatus, NodeResult, TimingInfo +from dbt.artifacts.schemas.base import ( + ArtifactMixin, + VersionedSchema, + schema_version, + BaseArtifactMetadata, +) from dbt_common.dataclass_schema import dbtClassMixin, StrEnum from dbt_common.exceptions import DbtInternalError diff --git a/core/dbt/artifacts/schemas/manifest/__init__.py b/core/dbt/artifacts/schemas/manifest/__init__.py new file mode 100644 index 00000000000..904f952cff1 --- /dev/null +++ b/core/dbt/artifacts/schemas/manifest/__init__.py @@ -0,0 +1,2 @@ +# alias to latest +from dbt.artifacts.schemas.manifest.v12.manifest import * # noqa diff --git a/core/dbt/artifacts/schemas/manifest/v12/__init__.py b/core/dbt/artifacts/schemas/manifest/v12/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/artifacts/schemas/manifest/v12/manifest.py b/core/dbt/artifacts/schemas/manifest/v12/manifest.py new file mode 100644 index 00000000000..806f6b35e0b --- /dev/null +++ b/core/dbt/artifacts/schemas/manifest/v12/manifest.py @@ -0,0 +1,180 @@ +from dataclasses import dataclass, field +from typing import Mapping, Iterable, Tuple, Optional, Dict, List, Any +from uuid import UUID + +from dbt.artifacts.schemas.base import ( + BaseArtifactMetadata, + ArtifactMixin, + schema_version, + get_artifact_schema_version, +) +from dbt.artifacts.schemas.upgrades import upgrade_manifest_json +from dbt.artifacts.resources import Documentation + +# TODO: remove usage of dbt modules other than dbt.artifacts +from dbt import tracking +from dbt.flags import get_flags +from dbt.contracts.graph.nodes import ( + Exposure, + GraphMemberNode, + Group, + Macro, + ManifestNode, + Metric, + SavedQuery, + SemanticModel, + SourceDefinition, + UnitTestDefinition, +) + + +NodeEdgeMap = Dict[str, List[str]] +UniqueID = str + + +@dataclass +class ManifestMetadata(BaseArtifactMetadata): + """Metadata for the manifest.""" + + dbt_schema_version: str = field( + default_factory=lambda: str(WritableManifest.dbt_schema_version) + ) + project_name: Optional[str] = field( + default=None, + metadata={ + "description": "Name of the root project", + }, + ) + project_id: Optional[str] = field( + default=None, + metadata={ + "description": "A unique identifier for the project, hashed from the project name", + }, + ) + user_id: Optional[UUID] = field( + default=None, + metadata={ + "description": "A unique identifier for the user", + }, + ) + send_anonymous_usage_stats: Optional[bool] = field( + default=None, + metadata=dict( + description=("Whether dbt is configured to send anonymous usage statistics") + ), + ) + adapter_type: Optional[str] = field( + default=None, + metadata=dict(description="The type name of the adapter"), + ) + + def __post_init__(self): + if tracking.active_user is None: + return + + if self.user_id is None: + self.user_id = tracking.active_user.id + + if self.send_anonymous_usage_stats is None: + self.send_anonymous_usage_stats = get_flags().SEND_ANONYMOUS_USAGE_STATS + + @classmethod + def default(cls): + return cls( + dbt_schema_version=str(WritableManifest.dbt_schema_version), + ) + + +@dataclass +@schema_version("manifest", 12) +class WritableManifest(ArtifactMixin): + nodes: Mapping[UniqueID, ManifestNode] = field( + metadata=dict(description=("The nodes defined in the dbt project and its dependencies")) + ) + sources: Mapping[UniqueID, SourceDefinition] = field( + metadata=dict(description=("The sources defined in the dbt project and its dependencies")) + ) + macros: Mapping[UniqueID, Macro] = field( + metadata=dict(description=("The macros defined in the dbt project and its dependencies")) + ) + docs: Mapping[UniqueID, Documentation] = field( + metadata=dict(description=("The docs defined in the dbt project and its dependencies")) + ) + exposures: Mapping[UniqueID, Exposure] = field( + metadata=dict( + description=("The exposures defined in the dbt project and its dependencies") + ) + ) + metrics: Mapping[UniqueID, Metric] = field( + metadata=dict(description=("The metrics defined in the dbt project and its dependencies")) + ) + groups: Mapping[UniqueID, Group] = field( + metadata=dict(description=("The groups defined in the dbt project")) + ) + selectors: Mapping[UniqueID, Any] = field( + metadata=dict(description=("The selectors defined in selectors.yml")) + ) + disabled: Optional[Mapping[UniqueID, List[GraphMemberNode]]] = field( + metadata=dict(description="A mapping of the disabled nodes in the target") + ) + parent_map: Optional[NodeEdgeMap] = field( + metadata=dict( + description="A mapping from child nodes to their dependencies", + ) + ) + child_map: Optional[NodeEdgeMap] = field( + metadata=dict( + description="A mapping from parent nodes to their dependents", + ) + ) + group_map: Optional[NodeEdgeMap] = field( + metadata=dict( + description="A mapping from group names to their nodes", + ) + ) + saved_queries: Mapping[UniqueID, SavedQuery] = field( + metadata=dict(description=("The saved queries defined in the dbt project")) + ) + semantic_models: Mapping[UniqueID, SemanticModel] = field( + metadata=dict(description=("The semantic models defined in the dbt project")) + ) + metadata: ManifestMetadata = field( + metadata=dict( + description="Metadata about the manifest", + ) + ) + unit_tests: Mapping[UniqueID, UnitTestDefinition] = field( + metadata=dict( + description="The unit tests defined in the project", + ) + ) + + @classmethod + def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]: + return [ + ("manifest", 4), + ("manifest", 5), + ("manifest", 6), + ("manifest", 7), + ("manifest", 8), + ("manifest", 9), + ("manifest", 10), + ("manifest", 11), + ] + + @classmethod + def upgrade_schema_version(cls, data): + """This overrides the "upgrade_schema_version" call in VersionedSchema (via + ArtifactMixin) to modify the dictionary passed in from earlier versions of the manifest.""" + manifest_schema_version = get_artifact_schema_version(data) + if manifest_schema_version <= 10: + data = upgrade_manifest_json(data, manifest_schema_version) + return cls.from_dict(data) + + def __post_serialize__(self, dct): + for unique_id, node in dct["nodes"].items(): + if "config_call_dict" in node: + del node["config_call_dict"] + if "defer_relation" in node: + del node["defer_relation"] + return dct diff --git a/core/dbt/artifacts/results.py b/core/dbt/artifacts/schemas/results.py similarity index 100% rename from core/dbt/artifacts/results.py rename to core/dbt/artifacts/schemas/results.py diff --git a/core/dbt/artifacts/schemas/run/__init__.py b/core/dbt/artifacts/schemas/run/__init__.py new file mode 100644 index 00000000000..5229dc92894 --- /dev/null +++ b/core/dbt/artifacts/schemas/run/__init__.py @@ -0,0 +1,2 @@ +# alias to latest +from dbt.artifacts.schemas.run.v5.run import * # noqa diff --git a/core/dbt/artifacts/schemas/run/v5/__init__.py b/core/dbt/artifacts/schemas/run/v5/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/artifacts/run.py b/core/dbt/artifacts/schemas/run/v5/run.py similarity index 98% rename from core/dbt/artifacts/run.py rename to core/dbt/artifacts/schemas/run/v5/run.py index d6479bb2009..eb731b71b5d 100644 --- a/core/dbt/artifacts/run.py +++ b/core/dbt/artifacts/schemas/run/v5/run.py @@ -6,13 +6,13 @@ from dbt.contracts.graph.nodes import CompiledNode -from dbt.artifacts.base import ( +from dbt.artifacts.schemas.base import ( BaseArtifactMetadata, ArtifactMixin, schema_version, get_artifact_schema_version, ) -from dbt.artifacts.results import ( +from dbt.artifacts.schemas.results import ( BaseResult, NodeResult, RunStatus, diff --git a/core/dbt/artifacts/schemas/upgrades/__init__.py b/core/dbt/artifacts/schemas/upgrades/__init__.py new file mode 100644 index 00000000000..d0839736d32 --- /dev/null +++ b/core/dbt/artifacts/schemas/upgrades/__init__.py @@ -0,0 +1 @@ +from dbt.artifacts.schemas.upgrades.upgrade_manifest import upgrade_manifest_json diff --git a/core/dbt/contracts/graph/manifest_upgrade.py b/core/dbt/artifacts/schemas/upgrades/upgrade_manifest.py similarity index 100% rename from core/dbt/contracts/graph/manifest_upgrade.py rename to core/dbt/artifacts/schemas/upgrades/upgrade_manifest.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 4e43561ff4b..ad2fbf61a77 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -17,8 +17,8 @@ DbtUsageException, ) from dbt.contracts.graph.manifest import Manifest -from dbt.artifacts.catalog import CatalogArtifact -from dbt.artifacts.run import RunExecutionResult +from dbt.artifacts.schemas.catalog import CatalogArtifact +from dbt.artifacts.schemas.run import RunExecutionResult from dbt_common.events.base_types import EventMsg from dbt.task.build import BuildTask from dbt.task.clean import CleanTask diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index bcde5bc83fd..4b7ed18258b 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -20,10 +20,8 @@ Generic, AbstractSet, ClassVar, - Iterable, ) from typing_extensions import Protocol -from uuid import UUID from dbt.contracts.graph.nodes import ( BaseNode, @@ -46,7 +44,9 @@ UnitTestFileFixture, ) from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersion -from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json + +# to preserve import paths +from dbt.artifacts.schemas.manifest import WritableManifest, ManifestMetadata, UniqueID from dbt.contracts.files import ( SourceFile, SchemaSourceFile, @@ -54,12 +54,6 @@ AnySourceFile, FixtureSourceFile, ) -from dbt.artifacts.base import ( - BaseArtifactMetadata, - ArtifactMixin, - schema_version, - get_artifact_schema_version, -) from dbt.contracts.util import SourceKey from dbt_common.dataclass_schema import dbtClassMixin @@ -73,19 +67,15 @@ from dbt_common.events.functions import fire_event from dbt_common.events.contextvars import get_node_info from dbt.events.types import MergedFromState, UnpinnedRefNewVersionAvailable -from dbt.node_types import NodeType, AccessType -from dbt.flags import get_flags +from dbt.node_types import NodeType, AccessType, REFABLE_NODE_TYPES, VERSIONED_NODE_TYPES from dbt.mp_context import get_mp_context -from dbt import tracking import dbt_common.utils import dbt_common.exceptions -NodeEdgeMap = Dict[str, List[str]] PackageName = str DocName = str RefName = str -UniqueID = str def find_unique_id_for_package(storage, key, package: Optional[PackageName]): @@ -171,8 +161,8 @@ def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SourceDef class RefableLookup(dbtClassMixin): # model, seed, snapshot - _lookup_types: ClassVar[set] = set(NodeType.refable()) - _versioned_types: ClassVar[set] = set(NodeType.versioned()) + _lookup_types: ClassVar[set] = set(REFABLE_NODE_TYPES) + _versioned_types: ClassVar[set] = set(VERSIONED_NODE_TYPES) def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} @@ -465,59 +455,6 @@ def _packages_to_search( return [current_project, node_package, None] -@dataclass -class ManifestMetadata(BaseArtifactMetadata): - """Metadata for the manifest.""" - - dbt_schema_version: str = field( - default_factory=lambda: str(WritableManifest.dbt_schema_version) - ) - project_name: Optional[str] = field( - default=None, - metadata={ - "description": "Name of the root project", - }, - ) - project_id: Optional[str] = field( - default=None, - metadata={ - "description": "A unique identifier for the project, hashed from the project name", - }, - ) - user_id: Optional[UUID] = field( - default=None, - metadata={ - "description": "A unique identifier for the user", - }, - ) - send_anonymous_usage_stats: Optional[bool] = field( - default=None, - metadata=dict( - description=("Whether dbt is configured to send anonymous usage statistics") - ), - ) - adapter_type: Optional[str] = field( - default=None, - metadata=dict(description="The type name of the adapter"), - ) - - def __post_init__(self): - if tracking.active_user is None: - return - - if self.user_id is None: - self.user_id = tracking.active_user.id - - if self.send_anonymous_usage_stats is None: - self.send_anonymous_usage_stats = get_flags().SEND_ANONYMOUS_USAGE_STATS - - @classmethod - def default(cls): - return cls( - dbt_schema_version=str(WritableManifest.dbt_schema_version), - ) - - def _sort_values(dct): """Given a dictionary, sort each value. This makes output deterministic, which helps for tests. @@ -1425,7 +1362,7 @@ def merge_from_artifact( Only non-ephemeral refable nodes are examined. """ - refables = set(NodeType.refable()) + refables = set(REFABLE_NODE_TYPES) merged = set() for unique_id, node in other.nodes.items(): current = self.nodes.get(unique_id) @@ -1457,7 +1394,6 @@ def merge_from_artifact( fire_event(MergedFromState(num_merged=len(merged), sample=sample)) # Methods that were formerly in ParseResult - def add_macro(self, source_file: SourceFile, macro: Macro): if macro.unique_id in self.macros: # detect that the macro exists and emit an error @@ -1645,101 +1581,6 @@ def __init__(self, macros) -> None: AnyManifest = Union[Manifest, MacroManifest] -@dataclass -@schema_version("manifest", 12) -class WritableManifest(ArtifactMixin): - nodes: Mapping[UniqueID, ManifestNode] = field( - metadata=dict(description=("The nodes defined in the dbt project and its dependencies")) - ) - sources: Mapping[UniqueID, SourceDefinition] = field( - metadata=dict(description=("The sources defined in the dbt project and its dependencies")) - ) - macros: Mapping[UniqueID, Macro] = field( - metadata=dict(description=("The macros defined in the dbt project and its dependencies")) - ) - docs: Mapping[UniqueID, Documentation] = field( - metadata=dict(description=("The docs defined in the dbt project and its dependencies")) - ) - exposures: Mapping[UniqueID, Exposure] = field( - metadata=dict( - description=("The exposures defined in the dbt project and its dependencies") - ) - ) - metrics: Mapping[UniqueID, Metric] = field( - metadata=dict(description=("The metrics defined in the dbt project and its dependencies")) - ) - groups: Mapping[UniqueID, Group] = field( - metadata=dict(description=("The groups defined in the dbt project")) - ) - selectors: Mapping[UniqueID, Any] = field( - metadata=dict(description=("The selectors defined in selectors.yml")) - ) - disabled: Optional[Mapping[UniqueID, List[GraphMemberNode]]] = field( - metadata=dict(description="A mapping of the disabled nodes in the target") - ) - parent_map: Optional[NodeEdgeMap] = field( - metadata=dict( - description="A mapping from child nodes to their dependencies", - ) - ) - child_map: Optional[NodeEdgeMap] = field( - metadata=dict( - description="A mapping from parent nodes to their dependents", - ) - ) - group_map: Optional[NodeEdgeMap] = field( - metadata=dict( - description="A mapping from group names to their nodes", - ) - ) - saved_queries: Mapping[UniqueID, SavedQuery] = field( - metadata=dict(description=("The saved queries defined in the dbt project")) - ) - semantic_models: Mapping[UniqueID, SemanticModel] = field( - metadata=dict(description=("The semantic models defined in the dbt project")) - ) - metadata: ManifestMetadata = field( - metadata=dict( - description="Metadata about the manifest", - ) - ) - unit_tests: Mapping[UniqueID, UnitTestDefinition] = field( - metadata=dict( - description="The unit tests defined in the project", - ) - ) - - @classmethod - def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]: - return [ - ("manifest", 4), - ("manifest", 5), - ("manifest", 6), - ("manifest", 7), - ("manifest", 8), - ("manifest", 9), - ("manifest", 10), - ("manifest", 11), - ] - - @classmethod - def upgrade_schema_version(cls, data): - """This overrides the "upgrade_schema_version" call in VersionedSchema (via - ArtifactMixin) to modify the dictionary passed in from earlier versions of the manifest.""" - manifest_schema_version = get_artifact_schema_version(data) - if manifest_schema_version <= 10: - data = upgrade_manifest_json(data, manifest_schema_version) - return cls.from_dict(data) - - def __post_serialize__(self, dct): - for unique_id, node in dct["nodes"].items(): - if "config_call_dict" in node: - del node["config_call_dict"] - if "defer_relation" in node: - del node["defer_relation"] - return dct - - def _check_duplicates(value: BaseNode, src: Mapping[str, BaseNode]): if value.unique_id in src: raise DuplicateResourceNameError(value, src[value.unique_id]) diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index f0e71303c5e..9b93e995cff 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -60,7 +60,13 @@ ) from dbt_common.events.contextvars import set_log_contextvars from dbt.flags import get_flags -from dbt.node_types import ModelLanguage, NodeType, AccessType +from dbt.node_types import ( + ModelLanguage, + NodeType, + AccessType, + REFABLE_NODE_TYPES, + VERSIONED_NODE_TYPES, +) from dbt_semantic_interfaces.references import ( EntityReference, MeasureReference, @@ -91,6 +97,8 @@ SavedQueryConfig, ) +from dbt.artifacts.resources import BaseArtifactNode, Documentation as DocumentationContract + # ===================================================================== # This contains the classes for all of the nodes and node-like objects @@ -116,16 +124,9 @@ @dataclass -class BaseNode(dbtClassMixin, Replaceable): +class BaseNode(BaseArtifactNode): """All nodes or node-like objects in this file should have this as a base class""" - name: str - resource_type: NodeType - package_name: str - path: str - original_file_path: str - unique_id: str - @property def search_name(self): return self.name @@ -136,7 +137,7 @@ def file_id(self): @property def is_refable(self): - return self.resource_type in NodeType.refable() + return self.resource_type in REFABLE_NODE_TYPES @property def should_store_failures(self): @@ -145,11 +146,11 @@ def should_store_failures(self): # will this node map to an object in the database? @property def is_relational(self): - return self.resource_type in NodeType.refable() + return self.resource_type in REFABLE_NODE_TYPES @property def is_versioned(self): - return self.resource_type in NodeType.versioned() and self.version is not None + return self.resource_type in VERSIONED_NODE_TYPES and self.version is not None @property def is_ephemeral(self): @@ -1178,10 +1179,7 @@ def depends_on_macros(self): @dataclass -class Documentation(BaseNode): - block_contents: str - resource_type: Literal[NodeType.Documentation] - +class Documentation(DocumentationContract, BaseNode): @property def search_name(self): return self.name diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index e0e2a37126f..5f355635872 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -309,8 +309,8 @@ class FreshnessThreshold(dbtClassMixin, Mergeable): error_after: Optional[Time] = field(default_factory=Time) filter: Optional[str] = None - def status(self, age: float) -> "dbt.artifacts.results.FreshnessStatus": # type: ignore # noqa F821 - from dbt.artifacts.results import FreshnessStatus + def status(self, age: float) -> "dbt.artifacts.schemas.results.FreshnessStatus": # type: ignore # noqa F821 + from dbt.artifacts.schemas.results import FreshnessStatus if self.error_after and self.error_after.exceeded(age): return FreshnessStatus.Error diff --git a/core/dbt/contracts/results.py b/core/dbt/contracts/results.py index ea69ee9b86d..c0775b10aa6 100644 --- a/core/dbt/contracts/results.py +++ b/core/dbt/contracts/results.py @@ -3,14 +3,14 @@ # This file is temporary, in order to not break various adapter tests, etc, until # they are updated to use the new locations. -from dbt.artifacts.base import ( +from dbt.artifacts.schemas.base import ( ArtifactMixin, BaseArtifactMetadata, VersionedSchema, schema_version, ) -from dbt.artifacts.results import ( +from dbt.artifacts.schemas.results import ( NodeStatus, RunStatus, TestStatus, @@ -23,7 +23,7 @@ ExecutionResult, ) -from dbt.artifacts.run import ( +from dbt.artifacts.schemas.run import ( RunResult, RunResultsMetadata, RunExecutionResult, @@ -31,7 +31,7 @@ process_run_result, ) -from dbt.artifacts.freshness import ( +from dbt.artifacts.schemas.freshness import ( FreshnessErrorEnum, FreshnessMetadata, FreshnessResult, @@ -45,7 +45,7 @@ PartialSourceFreshnessResult, ) -from dbt.artifacts.catalog import ( +from dbt.artifacts.schemas.catalog import ( CatalogResults, CatalogKey, StatsItem, diff --git a/core/dbt/contracts/sql.py b/core/dbt/contracts/sql.py index f2ea628b2eb..ec1033ef831 100644 --- a/core/dbt/contracts/sql.py +++ b/core/dbt/contracts/sql.py @@ -6,9 +6,9 @@ from dbt_common.dataclass_schema import dbtClassMixin from dbt.contracts.graph.nodes import ResultNode -from dbt.artifacts.results import TimingInfo, ExecutionResult -from dbt.artifacts.run import RunResult, RunResultsArtifact, RunExecutionResult -from dbt.artifacts.base import VersionedSchema, schema_version +from dbt.artifacts.schemas.results import TimingInfo, ExecutionResult +from dbt.artifacts.schemas.run import RunResult, RunResultsArtifact, RunExecutionResult +from dbt.artifacts.schemas.base import VersionedSchema, schema_version from dbt.logger import LogMessage diff --git a/core/dbt/contracts/state.py b/core/dbt/contracts/state.py index 6ee9b8636a4..9111e2dfb46 100644 --- a/core/dbt/contracts/state.py +++ b/core/dbt/contracts/state.py @@ -2,8 +2,8 @@ from typing import Optional from dbt.contracts.graph.manifest import WritableManifest -from dbt.artifacts.freshness import FreshnessExecutionResultArtifact -from dbt.artifacts.run import RunResultsArtifact +from dbt.artifacts.schemas.freshness import FreshnessExecutionResultArtifact +from dbt.artifacts.schemas.run import RunResultsArtifact from dbt_common.events.functions import fire_event from dbt.events.types import WarnStateTargetEqual from dbt.exceptions import IncompatibleSchemaError diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 0d5a543b433..25e4c055f14 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -14,7 +14,7 @@ DbtValidationError, CommandResultError, ) -from dbt.node_types import NodeType, AccessType +from dbt.node_types import NodeType, AccessType, REFABLE_NODE_TYPES from dbt_common.dataclass_schema import ValidationError @@ -1273,7 +1273,7 @@ def get_message(self) -> str: action = "looking for" # duplicate 'ref' targets - if node_type in NodeType.refable(): + if node_type in REFABLE_NODE_TYPES: formatted_name = f'ref("{duped_name}")' # duplicate sources elif node_type == NodeType.Source: diff --git a/core/dbt/node_types.py b/core/dbt/node_types.py index 19e8242c494..229c97352f3 100644 --- a/core/dbt/node_types.py +++ b/core/dbt/node_types.py @@ -1,84 +1,26 @@ from typing import List -from dbt_common.dataclass_schema import StrEnum - - -class AccessType(StrEnum): - Private = "private" - Protected = "protected" - Public = "public" - - @classmethod - def is_valid(cls, item): - try: - cls(item) - except ValueError: - return False - return True - - -class NodeType(StrEnum): - Model = "model" - Analysis = "analysis" - Test = "test" - Snapshot = "snapshot" - Operation = "operation" - Seed = "seed" - # TODO: rm? - RPCCall = "rpc" - SqlOperation = "sql_operation" - Documentation = "doc" - Source = "source" - Macro = "macro" - Exposure = "exposure" - Metric = "metric" - Group = "group" - SavedQuery = "saved_query" - SemanticModel = "semantic_model" - Unit = "unit_test" - Fixture = "fixture" - - @classmethod - def executable(cls) -> List["NodeType"]: - return [ - cls.Model, - cls.Test, - cls.Snapshot, - cls.Analysis, - cls.Operation, - cls.Seed, - cls.Documentation, - cls.RPCCall, - cls.SqlOperation, - ] - - @classmethod - def refable(cls) -> List["NodeType"]: - return [ - cls.Model, - cls.Seed, - cls.Snapshot, - ] - - @classmethod - def versioned(cls) -> List["NodeType"]: - return [ - cls.Model, - ] - - def pluralize(self) -> str: - if self is self.Analysis: - return "analyses" - elif self is self.SavedQuery: - return "saved_queries" - return f"{self}s" - - -class RunHookType(StrEnum): - Start = "on-run-start" - End = "on-run-end" - - -class ModelLanguage(StrEnum): - python = "python" - sql = "sql" +# preserving import path during dbt/artifacts refactor +from dbt.artifacts.resources.types import NodeType, AccessType, RunHookType, ModelLanguage # noqa + +EXECUTABLE_NODE_TYPES: List["NodeType"] = [ + NodeType.Model, + NodeType.Test, + NodeType.Snapshot, + NodeType.Analysis, + NodeType.Operation, + NodeType.Seed, + NodeType.Documentation, + NodeType.RPCCall, + NodeType.SqlOperation, +] + +REFABLE_NODE_TYPES: List["NodeType"] = [ + NodeType.Model, + NodeType.Seed, + NodeType.Snapshot, +] + +VERSIONED_NODE_TYPES: List["NodeType"] = [ + NodeType.Model, +] diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 9cf0e7f95a0..96d1a9ac5c2 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -109,7 +109,7 @@ NodeRelation, ) from dbt.contracts.graph.unparsed import NodeVersion -from dbt.artifacts.base import Writable +from dbt.artifacts.schemas.base import Writable from dbt.exceptions import ( TargetNotFoundError, AmbiguousAliasError, diff --git a/core/dbt/plugins/contracts.py b/core/dbt/plugins/contracts.py index 0ca9265ce8a..b0342c72a34 100644 --- a/core/dbt/plugins/contracts.py +++ b/core/dbt/plugins/contracts.py @@ -1,8 +1,8 @@ from typing import Dict # just exports, they need "noqa" so flake8 will not complain. -from dbt.artifacts.base import ArtifactMixin as PluginArtifact, schema_version # noqa -from dbt.artifacts.base import BaseArtifactMetadata # noqa +from dbt.artifacts.schemas.base import ArtifactMixin as PluginArtifact, schema_version # noqa +from dbt.artifacts.schemas.base import BaseArtifactMetadata # noqa from dbt_common.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin # noqa diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 3aa8b5f5d23..b86ac0f227e 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -16,9 +16,9 @@ from dbt.config.profile import read_profile from dbt.constants import DBT_PROJECT_FILE_NAME from dbt.contracts.graph.manifest import Manifest -from dbt.artifacts.results import TimingInfo, collect_timing_info -from dbt.artifacts.results import NodeStatus, RunningStatus, RunStatus -from dbt.artifacts.run import RunResult +from dbt.artifacts.schemas.results import TimingInfo, collect_timing_info +from dbt.artifacts.schemas.results import NodeStatus, RunningStatus, RunStatus +from dbt.artifacts.schemas.run import RunResult from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event from dbt.events.types import ( diff --git a/core/dbt/task/build.py b/core/dbt/task/build.py index 87d5b670cb6..e3ac5a6baa8 100644 --- a/core/dbt/task/build.py +++ b/core/dbt/task/build.py @@ -6,8 +6,8 @@ from .seed import SeedRunner as seed_runner from .test import TestRunner as test_runner -from dbt.artifacts.results import NodeStatus, RunStatus -from dbt.artifacts.run import RunResult +from dbt.artifacts.schemas.results import NodeStatus, RunStatus +from dbt.artifacts.schemas.run import RunResult from dbt.graph import ResourceTypeSelector, GraphQueue, Graph from dbt.node_types import NodeType from dbt.task.test import TestSelector diff --git a/core/dbt/task/clone.py b/core/dbt/task/clone.py index bc347763b19..49f7d857a30 100644 --- a/core/dbt/task/clone.py +++ b/core/dbt/task/clone.py @@ -5,11 +5,11 @@ from dbt.clients.jinja import MacroGenerator from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.manifest import WritableManifest -from dbt.artifacts.run import RunStatus, RunResult +from dbt.artifacts.schemas.run import RunStatus, RunResult from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.exceptions import DbtInternalError, CompilationError from dbt.graph import ResourceTypeSelector -from dbt.node_types import NodeType +from dbt.node_types import NodeType, REFABLE_NODE_TYPES from dbt.task.base import BaseRunner from dbt.task.run import _validate_materialization_relations_dict from dbt.task.runnable import GraphRunnableTask @@ -133,15 +133,15 @@ def before_run(self, adapter, selected_uids: AbstractSet[str]): @property def resource_types(self): if not self.args.resource_types: - return NodeType.refable() + return REFABLE_NODE_TYPES values = set(self.args.resource_types) if "all" in values: values.remove("all") - values.update(NodeType.refable()) + values.update(REFABLE_NODE_TYPES) - values = [NodeType(val) for val in values if val in NodeType.refable()] + values = [NodeType(val) for val in values if val in REFABLE_NODE_TYPES] return list(values) diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py index 05214139e06..4541dfd1567 100644 --- a/core/dbt/task/compile.py +++ b/core/dbt/task/compile.py @@ -1,6 +1,6 @@ import threading -from dbt.artifacts.run import RunStatus, RunResult +from dbt.artifacts.schemas.run import RunStatus, RunResult from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.events.types import Note @@ -12,7 +12,7 @@ ) from dbt.graph import ResourceTypeSelector -from dbt.node_types import NodeType +from dbt.node_types import NodeType, EXECUTABLE_NODE_TYPES from dbt.parser.manifest import process_node from dbt.parser.sql import SqlBlockParser from dbt.task.base import BaseRunner @@ -54,7 +54,7 @@ def get_node_selector(self) -> ResourceTypeSelector: if getattr(self.args, "inline", None): resource_types = [NodeType.SqlOperation] else: - resource_types = NodeType.executable() + resource_types = EXECUTABLE_NODE_TYPES if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") diff --git a/core/dbt/task/debug.py b/core/dbt/task/debug.py index 15507ba0fc2..ea0f636bd6c 100644 --- a/core/dbt/task/debug.py +++ b/core/dbt/task/debug.py @@ -21,7 +21,7 @@ from dbt.adapters.factory import get_adapter, register_adapter from dbt.config import PartialProject, Project, Profile from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer -from dbt.artifacts.results import RunStatus +from dbt.artifacts.schemas.results import RunStatus from dbt.clients.yaml_helper import load_yaml_text from dbt.links import ProfileConfigDocs from dbt_common.ui import green, red diff --git a/core/dbt/task/docs/generate.py b/core/dbt/task/docs/generate.py index a3d8cdf5e53..acd68cb37ee 100644 --- a/core/dbt/task/docs/generate.py +++ b/core/dbt/task/docs/generate.py @@ -15,8 +15,8 @@ from dbt.adapters.factory import get_adapter from dbt.contracts.graph.nodes import ResultNode from dbt.contracts.graph.manifest import Manifest -from dbt.artifacts.results import NodeStatus -from dbt.artifacts.catalog import ( +from dbt.artifacts.schemas.results import NodeStatus +from dbt.artifacts.schemas.catalog import ( TableMetadata, CatalogTable, CatalogResults, @@ -30,7 +30,7 @@ from dbt_common.exceptions import DbtInternalError from dbt.exceptions import AmbiguousCatalogMatchError from dbt.graph import ResourceTypeSelector -from dbt.node_types import NodeType +from dbt.node_types import EXECUTABLE_NODE_TYPES from dbt_common.events.functions import fire_event from dbt.adapters.events.types import ( WriteCatalogFailure, @@ -342,7 +342,7 @@ def get_node_selector(self) -> ResourceTypeSelector: graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, - resource_types=NodeType.executable(), + resource_types=EXECUTABLE_NODE_TYPES, ) def get_catalog_results( diff --git a/core/dbt/task/freshness.py b/core/dbt/task/freshness.py index 0ec3157a596..c506c54e013 100644 --- a/core/dbt/task/freshness.py +++ b/core/dbt/task/freshness.py @@ -9,7 +9,7 @@ ) from .run import RunTask -from dbt.artifacts.freshness import ( +from dbt.artifacts.schemas.freshness import ( FreshnessResult, PartialSourceFreshnessResult, SourceFreshnessResult, diff --git a/core/dbt/task/printer.py b/core/dbt/task/printer.py index a3316edd144..dc8f4c483d1 100644 --- a/core/dbt/task/printer.py +++ b/core/dbt/task/printer.py @@ -20,7 +20,7 @@ from dbt.tracking import InvocationProcessor from dbt_common.events.format import pluralize -from dbt.artifacts.results import NodeStatus +from dbt.artifacts.schemas.results import NodeStatus from dbt.node_types import NodeType diff --git a/core/dbt/task/retry.py b/core/dbt/task/retry.py index b2987896973..717f4f90ec9 100644 --- a/core/dbt/task/retry.py +++ b/core/dbt/task/retry.py @@ -6,7 +6,7 @@ from dbt.flags import set_flags, get_flags from dbt.cli.types import Command as CliCommand from dbt.config import RuntimeConfig -from dbt.artifacts.results import NodeStatus +from dbt.artifacts.schemas.results import NodeStatus from dbt.contracts.state import load_result_state from dbt_common.exceptions import DbtRuntimeError from dbt.graph import GraphQueue diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 0ef9d92a1fe..aa8407694c1 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -19,8 +19,8 @@ from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.model_config import Hook from dbt.contracts.graph.nodes import HookNode, ResultNode -from dbt.artifacts.results import NodeStatus, RunStatus, RunningStatus, BaseResult -from dbt.artifacts.run import RunResult +from dbt.artifacts.schemas.results import NodeStatus, RunStatus, RunningStatus, BaseResult +from dbt.artifacts.schemas.run import RunResult from dbt.exceptions import ( CompilationError, DbtInternalError, diff --git a/core/dbt/task/run_operation.py b/core/dbt/task/run_operation.py index 09c2d409eae..7b217853ce7 100644 --- a/core/dbt/task/run_operation.py +++ b/core/dbt/task/run_operation.py @@ -9,8 +9,8 @@ from dbt.adapters.factory import get_adapter from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import HookNode -from dbt.artifacts.results import RunStatus, TimingInfo -from dbt.artifacts.run import RunResultsArtifact, RunResult +from dbt.artifacts.schemas.results import RunStatus, TimingInfo +from dbt.artifacts.schemas.run import RunResultsArtifact, RunResult from dbt_common.events.functions import fire_event from dbt.events.types import ( LogDebugStackTrace, diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index b271631f063..145d5778784 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -15,8 +15,8 @@ from dbt.adapters.factory import get_adapter from dbt.contracts.graph.manifest import WritableManifest from dbt.contracts.graph.nodes import ResultNode -from dbt.artifacts.results import NodeStatus, RunningStatus, RunStatus, BaseResult -from dbt.artifacts.run import RunExecutionResult, RunResult +from dbt.artifacts.schemas.results import NodeStatus, RunningStatus, RunStatus, BaseResult +from dbt.artifacts.schemas.run import RunExecutionResult, RunResult from dbt.contracts.state import PreviousState from dbt_common.events.contextvars import log_contextvars, task_contextvars from dbt_common.events.functions import fire_event, warn_or_error diff --git a/core/dbt/task/seed.py b/core/dbt/task/seed.py index 419891524ed..ce8ba0f35b9 100644 --- a/core/dbt/task/seed.py +++ b/core/dbt/task/seed.py @@ -5,7 +5,7 @@ print_run_end_messages, ) -from dbt.artifacts.results import RunStatus, NodeStatus +from dbt.artifacts.schemas.results import RunStatus, NodeStatus from dbt_common.exceptions import DbtInternalError from dbt.graph import ResourceTypeSelector from dbt.logger import TextOnly diff --git a/core/dbt/task/show.py b/core/dbt/task/show.py index c7789262361..608ba6fdebd 100644 --- a/core/dbt/task/show.py +++ b/core/dbt/task/show.py @@ -4,7 +4,7 @@ from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.nodes import SeedNode -from dbt.artifacts.run import RunResult, RunStatus +from dbt.artifacts.schemas.run import RunResult, RunStatus from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.events.types import Note diff --git a/core/dbt/task/snapshot.py b/core/dbt/task/snapshot.py index 8b35837158e..728ea49851c 100644 --- a/core/dbt/task/snapshot.py +++ b/core/dbt/task/snapshot.py @@ -6,7 +6,7 @@ from dbt.events.types import LogSnapshotResult from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType -from dbt.artifacts.results import NodeStatus +from dbt.artifacts.schemas.results import NodeStatus from dbt_common.utils import cast_dict_to_dict_of_strings diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index ab5f439be9e..02ff16231bb 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -17,9 +17,9 @@ from dbt.contracts.graph.nodes import TestNode, UnitTestDefinition, UnitTestNode from dbt.contracts.graph.manifest import Manifest -from dbt.artifacts.results import TestStatus -from dbt.artifacts.run import RunResult -from dbt.artifacts.catalog import PrimitiveDict +from dbt.artifacts.schemas.results import TestStatus +from dbt.artifacts.schemas.run import RunResult +from dbt.artifacts.schemas.catalog import PrimitiveDict from dbt.context.providers import generate_runtime_model_context from dbt.clients.jinja import MacroGenerator from dbt_common.events.functions import fire_event diff --git a/scripts/collect-artifact-schema.py b/scripts/collect-artifact-schema.py index 255022f9658..5bedbfbbd6b 100755 --- a/scripts/collect-artifact-schema.py +++ b/scripts/collect-artifact-schema.py @@ -6,10 +6,10 @@ from typing import Type, Dict, Any from dbt.contracts.graph.manifest import WritableManifest -from dbt.artifacts.catalog import CatalogArtifact -from dbt.artifacts.run import RunResultsArtifact -from dbt.artifacts.freshness import FreshnessExecutionResultArtifact -from dbt.artifacts.base import VersionedSchema +from dbt.artifacts.schemas.catalog import CatalogArtifact +from dbt.artifacts.schemas.run import RunResultsArtifact +from dbt.artifacts.schemas.freshness import FreshnessExecutionResultArtifact +from dbt.artifacts.schemas.base import VersionedSchema from dbt_common.clients.system import write_file diff --git a/tests/adapter/dbt/tests/adapter/basic/test_incremental.py b/tests/adapter/dbt/tests/adapter/basic/test_incremental.py index fbba4bf7c31..3e1b83794f6 100644 --- a/tests/adapter/dbt/tests/adapter/basic/test_incremental.py +++ b/tests/adapter/dbt/tests/adapter/basic/test_incremental.py @@ -1,6 +1,6 @@ import pytest from dbt.tests.util import run_dbt, check_relations_equal, relation_from_name -from dbt.artifacts.results import RunStatus +from dbt.artifacts.schemas.results import RunStatus from dbt.tests.adapter.basic.files import ( seeds_base_csv, seeds_added_csv, diff --git a/tests/adapter/dbt/tests/adapter/catalog/relation_types.py b/tests/adapter/dbt/tests/adapter/catalog/relation_types.py index f67f9c4718a..ccad99fde39 100644 --- a/tests/adapter/dbt/tests/adapter/catalog/relation_types.py +++ b/tests/adapter/dbt/tests/adapter/catalog/relation_types.py @@ -1,4 +1,4 @@ -from dbt.artifacts.catalog import CatalogArtifact +from dbt.artifacts.schemas.catalog import CatalogArtifact from dbt.tests.util import run_dbt import pytest diff --git a/tests/adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py index 01f9c2da235..e08e332ad3e 100644 --- a/tests/adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py +++ b/tests/adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py @@ -1,6 +1,6 @@ import pytest from dbt.tests.util import run_dbt, check_relations_equal -from dbt.artifacts.results import RunStatus +from dbt.artifacts.schemas.results import RunStatus from collections import namedtuple from pathlib import Path diff --git a/tests/adapter/dbt/tests/adapter/store_test_failures_tests/basic.py b/tests/adapter/dbt/tests/adapter/store_test_failures_tests/basic.py index a4158544766..cd8f5a84691 100644 --- a/tests/adapter/dbt/tests/adapter/store_test_failures_tests/basic.py +++ b/tests/adapter/dbt/tests/adapter/store_test_failures_tests/basic.py @@ -3,7 +3,7 @@ import pytest -from dbt.artifacts.results import TestStatus +from dbt.artifacts.schemas.results import TestStatus from dbt.tests.util import run_dbt, check_relation_types from dbt.tests.adapter.store_test_failures_tests import _files diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index cc92997b48a..7aadb308513 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -17,8 +17,8 @@ ) from dbt.contracts.graph.manifest import WritableManifest -from dbt.artifacts.results import RunStatus -from dbt.artifacts.run import RunResultsArtifact +from dbt.artifacts.schemas.results import RunStatus +from dbt.artifacts.schemas.run import RunResultsArtifact models__schema_yml = """ version: 2 diff --git a/tests/functional/artifacts/test_previous_version_state.py b/tests/functional/artifacts/test_previous_version_state.py index d6ca153550b..e8eb3b251c8 100644 --- a/tests/functional/artifacts/test_previous_version_state.py +++ b/tests/functional/artifacts/test_previous_version_state.py @@ -4,9 +4,9 @@ import pytest -from dbt.artifacts.base import get_artifact_schema_version +from dbt.artifacts.schemas.base import get_artifact_schema_version from dbt.contracts.graph.manifest import WritableManifest -from dbt.artifacts.run import RunResultsArtifact +from dbt.artifacts.schemas.run import RunResultsArtifact from dbt.exceptions import IncompatibleSchemaError from dbt.tests.util import run_dbt, get_manifest diff --git a/tests/unit/test_contracts_graph_unparsed.py b/tests/unit/test_contracts_graph_unparsed.py index 72839ea5bb2..2a6d55a20c7 100644 --- a/tests/unit/test_contracts_graph_unparsed.py +++ b/tests/unit/test_contracts_graph_unparsed.py @@ -26,7 +26,7 @@ UnparsedMetricInputMeasure, UnparsedVersion, ) -from dbt.artifacts.results import FreshnessStatus +from dbt.artifacts.schemas.results import FreshnessStatus from dbt.node_types import NodeType from .utils import ContractTestCase diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 9e8355151f3..1c40ad9a39f 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -7,8 +7,8 @@ from dbt.adapters.events import types as adapter_types from dbt_common.events.event_manager_client import ctx_set_event_manager -from dbt.artifacts.results import TimingInfo, RunStatus -from dbt.artifacts.run import RunResult +from dbt.artifacts.schemas.results import TimingInfo, RunStatus +from dbt.artifacts.schemas.run import RunResult from dbt_common.events import types from dbt.adapters.events.logging import AdapterLogger from dbt_common.events.base_types import msg_from_base_event