consolidate cross-project ref entrypoint + plugin framework (#7955)

.changes/unreleased/Under the Hood-20230627-222328.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: 'Refactor: entry point for cross-project ref'
+time: 2023-06-27T22:23:28.73069-04:00
+custom:
+  Author: michelleark
+  Issue: "7954"

core/dbt/cli/main.py

@@ -76,7 +76,6 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
             dbt_ctx.obj = {
                 "manifest": self.manifest,
                 "callbacks": self.callbacks,
-                "_publications": kwargs.get("publications"),
             }
 
             for key, value in kwargs.items():

core/dbt/cli/requires.py

@@ -23,6 +23,7 @@
 from dbt.profiler import profiler
 from dbt.tracking import active_user, initialize_from_flags, track_run
 from dbt.utils import cast_dict_to_dict_of_strings
+from dbt.plugins import set_up_plugin_manager, get_plugin_manager
 
 from click import Context
 from functools import update_wrapper
@@ -160,6 +161,9 @@ def wrapper(*args, **kwargs):
         )
         ctx.obj["project"] = project
 
+        # Plugins
+        set_up_plugin_manager(project_name=project.project_name)
+
         if dbt.tracking.active_user is not None:
             project_id = None if project is None else project.hashed_name()
 
@@ -242,12 +246,15 @@ def wrapper(*args, **kwargs):
                 manifest = ManifestLoader.get_full_manifest(
                     runtime_config,
                     write_perf_info=write_perf_info,
-                    publications=ctx.obj.get("_publications"),
                 )
 
                 ctx.obj["manifest"] = manifest
                 if write and ctx.obj["flags"].write_json:
-                    write_manifest(manifest, ctx.obj["runtime_config"].project_target_path)
+                    write_manifest(manifest, runtime_config.project_target_path)
+                    pm = get_plugin_manager(runtime_config.project_name)
+                    plugin_artifacts = pm.get_manifest_artifacts(manifest)
+                    for path, plugin_artifact in plugin_artifacts.items():
+                        plugin_artifact.write(path)
 
             return func(*args, **kwargs)
 

core/dbt/config/project.py

@@ -39,7 +39,6 @@
     SemverString,
 )
 from dbt.contracts.project import PackageConfig, ProjectPackageMetadata
-from dbt.contracts.publication import ProjectDependencies
 from dbt.dataclass_schema import ValidationError
 from .renderer import DbtProjectYamlRenderer, PackageRenderer
 from .selectors import (
@@ -115,16 +114,13 @@ def package_and_project_data_from_root(project_root):
 
     packages_specified_path = PACKAGES_FILE_NAME
     packages_dict = {}
-    dependent_projects_dict = {}
     if "packages" in dependencies_yml_dict:
         packages_dict["packages"] = dependencies_yml_dict["packages"]
         packages_specified_path = DEPENDENCIES_FILE_NAME
     else:  # don't check for "packages" here so we capture invalid keys in packages.yml
         packages_dict = packages_yml_dict
-    if "projects" in dependencies_yml_dict:
-        dependent_projects_dict["projects"] = dependencies_yml_dict["projects"]
 
-    return packages_dict, dependent_projects_dict, packages_specified_path
+    return packages_dict, packages_specified_path
 
 
 def package_config_from_data(packages_data: Dict[str, Any]) -> PackageConfig:
@@ -139,21 +135,6 @@ def package_config_from_data(packages_data: Dict[str, Any]) -> PackageConfig:
     return packages
 
 
-def dependent_project_config_from_data(
-    dependent_projects_data: Dict[str, Any]
-) -> ProjectDependencies:
-    if not dependent_projects_data:
-        dependent_projects_data = {"projects": []}
-
-    try:
-        ProjectDependencies.validate(dependent_projects_data)
-        dependent_projects = ProjectDependencies.from_dict(dependent_projects_data)
-    except ValidationError as e:
-        msg = f"Malformed dependencies.yml: {e}"
-        raise DbtProjectError(msg)
-    return dependent_projects
-
-
 def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]:
     """Parse multiple versions as read from disk. The versions value may be any
     one of:
@@ -278,9 +259,6 @@ def _get_required_version(
 class RenderComponents:
     project_dict: Dict[str, Any] = field(metadata=dict(description="The project dictionary"))
     packages_dict: Dict[str, Any] = field(metadata=dict(description="The packages dictionary"))
-    dependent_projects_dict: Dict[str, Any] = field(
-        metadata=dict(description="The dependent projects dictionary")
-    )
     selectors_dict: Dict[str, Any] = field(metadata=dict(description="The selectors dictionary"))
 
 
@@ -321,15 +299,11 @@ def get_rendered(
         rendered_packages = renderer.render_packages(
             self.packages_dict, self.packages_specified_path
         )
-        rendered_dependent_projects = renderer.render_dependent_projects(
-            self.dependent_projects_dict
-        )
         rendered_selectors = renderer.render_selectors(self.selectors_dict)
 
         return RenderComponents(
             project_dict=rendered_project,
             packages_dict=rendered_packages,
-            dependent_projects_dict=rendered_dependent_projects,
             selectors_dict=rendered_selectors,
         )
 
@@ -376,7 +350,6 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         unrendered = RenderComponents(
             project_dict=self.project_dict,
             packages_dict=self.packages_dict,
-            dependent_projects_dict=self.dependent_projects_dict,
             selectors_dict=self.selectors_dict,
         )
         dbt_version = _get_required_version(
@@ -478,9 +451,6 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         query_comment = _query_comment_from_cfg(cfg.query_comment)
 
         packages: PackageConfig = package_config_from_data(rendered.packages_dict)
-        dependent_projects: ProjectDependencies = dependent_project_config_from_data(
-            rendered.dependent_projects_dict
-        )
         selectors = selector_config_from_data(rendered.selectors_dict)
         manifest_selectors: Dict[str, Any] = {}
         if rendered.selectors_dict and rendered.selectors_dict["selectors"]:
@@ -516,7 +486,6 @@ def create_project(self, rendered: RenderComponents) -> "Project":
             snapshots=snapshots,
             dbt_version=dbt_version,
             packages=packages,
-            dependent_projects=dependent_projects,
             manifest_selectors=manifest_selectors,
             selectors=selectors,
             query_comment=query_comment,
@@ -539,7 +508,6 @@ def from_dicts(
         project_root: str,
         project_dict: Dict[str, Any],
         packages_dict: Dict[str, Any],
-        dependent_projects_dict: Dict[str, Any],
         selectors_dict: Dict[str, Any],
         *,
         verify_version: bool = False,
@@ -556,7 +524,6 @@ def from_dicts(
             project_root=project_root,
             project_dict=project_dict,
             packages_dict=packages_dict,
-            dependent_projects_dict=dependent_projects_dict,
             selectors_dict=selectors_dict,
             verify_version=verify_version,
             packages_specified_path=packages_specified_path,
@@ -570,7 +537,6 @@ def from_project_root(
         project_dict = load_raw_project(project_root)
         (
             packages_dict,
-            dependent_projects_dict,
             packages_specified_path,
         ) = package_and_project_data_from_root(project_root)
         selectors_dict = selector_data_from_root(project_root)
@@ -579,7 +545,6 @@ def from_project_root(
             project_dict=project_dict,
             selectors_dict=selectors_dict,
             packages_dict=packages_dict,
-            dependent_projects_dict=dependent_projects_dict,
             verify_version=verify_version,
             packages_specified_path=packages_specified_path,
         )
@@ -636,7 +601,6 @@ class Project:
     vars: VarProvider
     dbt_version: List[VersionSpecifier]
     packages: PackageConfig
-    dependent_projects: ProjectDependencies
     manifest_selectors: Dict[str, Any]
     selectors: SelectorConfig
     query_comment: QueryComment
@@ -727,13 +691,6 @@ def validate(self):
         except ValidationError as e:
             raise ProjectContractBrokenError(e) from e
 
-    @classmethod
-    def partial_load(cls, project_root: str, *, verify_version: bool = False) -> PartialProject:
-        return PartialProject.from_project_root(
-            project_root,
-            verify_version=verify_version,
-        )
-
     # Called by:
     # RtConfig.load_dependencies => RtConfig.load_projects => RtConfig.new_project => Project.from_project_root
     # RtConfig.from_args => RtConfig.collect_parts => load_project => Project.from_project_root

core/dbt/config/renderer.py

@@ -142,10 +142,6 @@ def render_packages(self, packages: Dict[str, Any], packages_specified_path: str
         else:
             return package_renderer.render_data(packages)
 
-    def render_dependent_projects(self, dependent_projects: Dict[str, Any]):
-        """This is a no-op to maintain regularity in the interfaces. We don't render dependencies.yml."""
-        return dependent_projects
-
     def render_selectors(self, selectors: Dict[str, Any]):
         return self.render_data(selectors)
 

core/dbt/config/runtime.py

@@ -161,7 +161,6 @@ def from_parts(
             snapshots=project.snapshots,
             dbt_version=project.dbt_version,
             packages=project.packages,
-            dependent_projects=project.dependent_projects,
             manifest_selectors=project.manifest_selectors,
             selectors=project.selectors,
             query_comment=project.query_comment,

core/dbt/contracts/graph/manifest.py

@@ -22,8 +22,6 @@
 from typing_extensions import Protocol
 from uuid import UUID
 
-from dbt.contracts.publication import PublicationConfig
-
 from dbt.contracts.graph.nodes import (
     BaseNode,
     Documentation,
@@ -700,7 +698,6 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
     source_patches: MutableMapping[SourceKey, SourcePatch] = field(default_factory=dict)
     disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict)
     env_vars: MutableMapping[str, str] = field(default_factory=dict)
-    publications: MutableMapping[str, PublicationConfig] = field(default_factory=dict)
     semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict)
 
     _doc_lookup: Optional[DocLookup] = field(
@@ -851,7 +848,6 @@ def deepcopy(self):
             disabled={k: _deepcopy(v) for k, v in self.disabled.items()},
             files={k: _deepcopy(v) for k, v in self.files.items()},
             state_check=_deepcopy(self.state_check),
-            publications={k: _deepcopy(v) for k, v in self.publications.items()},
             semantic_models={k: _deepcopy(v) for k, v in self.semantic_models.items()},
         )
         copy.build_flat_graph()
@@ -1297,7 +1293,6 @@ def __reduce_ex__(self, protocol):
             self.source_patches,
             self.disabled,
             self.env_vars,
-            self.publications,
             self.semantic_models,
             self._doc_lookup,
             self._source_lookup,

core/dbt/contracts/graph/node_args.py

@@ -3,7 +3,7 @@
 from typing import Optional, List
 
 from dbt.contracts.graph.unparsed import NodeVersion
-from dbt.node_types import AccessType
+from dbt.node_types import NodeType, AccessType
 
 
 @dataclass
@@ -21,3 +21,11 @@ class ModelNodeArgs:
     generated_at: datetime = field(default_factory=datetime.utcnow)
     depends_on_nodes: List[str] = field(default_factory=list)
     enabled: bool = True
+
+    @property
+    def unique_id(self):
+        unique_id = f"{NodeType.Model}.{self.package_name}.{self.name}"
+        if self.version:
+            unique_id = f"{unique_id}.{self.verison}"
+
+        return unique_id

core/dbt/contracts/graph/nodes.py

@@ -581,9 +581,7 @@ class ModelNode(CompiledNode):
 
     @classmethod
     def from_args(cls, args: ModelNodeArgs) -> "ModelNode":
-        unique_id = f"{NodeType.Model}.{args.package_name}.{args.name}"
-        if args.version:
-            unique_id = f"{unique_id}.{args.version}"
+        unique_id = args.unique_id
 
         # build unrendered config -- for usage in ParsedNode.same_contents
         unrendered_config = {}

core/dbt/events/types.proto

@@ -1517,18 +1517,6 @@ message NoNodesForSelectionCriteriaMsg {
     NoNodesForSelectionCriteria data = 2;
 }
 
-// P - Artifacts
-
-// P001
-message PublicationArtifactAvailable {
-    google.protobuf.Struct pub_artifact = 1;
-}
-
-message PublicationArtifactAvailableMsg {
-    EventInfo info = 1;
-    PublicationArtifactAvailable data = 2;
-}
-
 // Q - Node execution
 
 // Q001