deprecate materialization overrides from imported packages (#9971)

.changes/unreleased/Under the Hood-20240418-172528.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Raise deprecation warning if installed package overrides built-in materialization
+time: 2024-04-18T17:25:28.37886-04:00
+custom:
+  Author: michelleark
+  Issue: "9971"

core/dbt/contracts/graph/manifest.py

@@ -23,7 +23,6 @@
 )
 from typing_extensions import Protocol
 from uuid import UUID
-
 from dbt.contracts.graph.nodes import (
     BaseNode,
     Documentation,
@@ -59,7 +58,7 @@
 from dbt.events.contextvars import get_node_info
 from dbt.node_types import NodeType, AccessType
 from dbt.flags import get_flags, MP_CONTEXT
-from dbt import tracking
+from dbt import tracking, deprecations
 import dbt.utils
 
 
@@ -562,11 +561,15 @@ def __lt__(self, other: object) -> bool:
 
 
 class CandidateList(List[M]):
-    def last(self) -> Optional[Macro]:
+    def last_candidate(self) -> Optional[MacroCandidate]:
         if not self:
             return None
         self.sort()
-        return self[-1].macro
+        return self[-1]
+
+    def last(self) -> Optional[Macro]:
+        last_candidate = self.last_candidate()
+        return last_candidate.macro if last_candidate is not None else None
 
 
 def _get_locality(macro: Macro, root_project_name: str, internal_packages: Set[str]) -> Locality:
@@ -850,7 +853,24 @@ def find_materialization_macro_by_name(
                 for specificity, atype in enumerate(self._get_parent_adapter_types(adapter_type))
             )
         )
-        return candidates.last()
+        core_candidates = [
+            candidate for candidate in candidates if candidate.locality == Locality.Core
+        ]
+
+        materialization_candidate = candidates.last_candidate()
+        # If an imported materialization macro was found that also had a core candidate, fire a deprecation
+        if (
+            materialization_candidate is not None
+            and materialization_candidate.locality == Locality.Imported
+            and core_candidates
+        ):
+            deprecations.warn(
+                "package-materialization-override",
+                package_name=materialization_candidate.macro.package_name,
+                materialization_name=materialization_name,
+            )
+
+        return materialization_candidate.macro if materialization_candidate else None
 
     def get_resource_fqns(self) -> Mapping[str, PathSet]:
         resource_fqns: Dict[str, Set[Tuple[str, ...]]] = {}

core/dbt/deprecations.py

@@ -96,6 +96,11 @@ class CollectFreshnessReturnSignature(DBTDeprecation):
     _event = "CollectFreshnessReturnSignature"
 
 
+class PackageMaterializationOverrideDeprecation(DBTDeprecation):
+    _name = "package-materialization-override"
+    _event = "PackageMaterializationOverrideDeprecation"
+
+
 def renamed_env_var(old_name: str, new_name: str):
     class EnvironmentVariableRenamed(DBTDeprecation):
         _name = f"environment-variable-renamed:{old_name}"
@@ -134,6 +139,7 @@ def warn(name, *args, **kwargs):
     ConfigLogPathDeprecation(),
     ConfigTargetPathDeprecation(),
     CollectFreshnessReturnSignature(),
+    PackageMaterializationOverrideDeprecation(),
 ]
 
 deprecations: Dict[str, DBTDeprecation] = {d.name: d for d in deprecations_list}

core/dbt/events/types.proto

@@ -394,6 +394,17 @@ message CollectFreshnessReturnSignatureMsg {
     CollectFreshnessReturnSignature data = 2;
 }
 
+// D016
+message PackageMaterializationOverrideDeprecation {
+    string package_name = 1;
+    string materialization_name = 2;
+}
+
+message PackageMaterializationOverrideDeprecationMsg {
+    EventInfo info = 1;
+    PackageMaterializationOverrideDeprecation data = 2;
+}
+
 // E - DB Adapter
 
 // E001

core/dbt/events/types.py

@@ -797,6 +797,16 @@ def message(self) -> str:
         return line_wrap_message(warning_tag(msg))
 
 
+class PackageMaterializationOverrideDeprecation(WarnLevel):
+    def code(self) -> str:
+        return "D016"
+
+    def message(self) -> str:
+        description = f"Installed package '{self.package_name}' is overriding the built-in materialization '{self.materialization_name}'. Overrides of built-in materializations from installed packages will be deprecated in future versions of dbt. Please refer to https://docs.getdbt.com/reference/global-configs/legacy-behaviors#deprecate_package_materialization_builtin_override for detailed documentation and suggested workarounds."
+
+        return line_wrap_message(warning_tag(description))
+
+
 # =======================================================
 # I - Project parsing
 # =======================================================

tests/functional/materializations/conftest.py

@@ -325,6 +325,21 @@
 {%- endmaterialization -%}
 """
 
+custom_materialization_dep__dbt_project_yml = """
+name: custom_materialization_default
+macro-paths: ['macros']
+"""
+
+custom_materialization_sql = """
+{% materialization custom_materialization, default %}
+    {%- set target_relation = this.incorporate(type='table') %}
+    {% call statement('main') -%}
+        select 1 as column1
+    {%- endcall %}
+    {{ return({'relations': [target_relation]}) }}
+{% endmaterialization %}
+"""
+
 
 @pytest.fixture(scope="class")
 def override_view_adapter_pass_dep(project_root):
@@ -368,3 +383,12 @@ def override_view_return_no_relation(project_root):
         },
     }
     write_project_files(project_root, "override-view-return-no-relation", files)
+
+
+@pytest.fixture(scope="class")
+def custom_materialization_dep(project_root):
+    files = {
+        "dbt_project.yml": custom_materialization_dep__dbt_project_yml,
+        "macros": {"custom_materialization.sql": custom_materialization_sql},
+    }
+    write_project_files(project_root, "custom-materialization-dep", files)

tests/functional/materializations/test_custom_materialization.py

@@ -1,7 +1,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-
+from dbt import deprecations
 
 models__model_sql = """
 {{ config(materialized='view') }}
@@ -10,34 +10,100 @@
 """
 
 
+models_custom_materialization__model_sql = """
+{{ config(materialized='custom_materialization') }}
+select 1 as id
+
+"""
+
+
 @pytest.fixture(scope="class")
 def models():
     return {"model.sql": models__model_sql}
 
 
+@pytest.fixture(scope="class")
+def set_up_deprecations():
+    deprecations.reset_deprecations()
+    assert deprecations.active_deprecations == set()
+
+
 class TestOverrideAdapterDependency:
     # make sure that if there's a dependency with an adapter-specific
     # materialization, we honor that materialization
     @pytest.fixture(scope="class")
     def packages(self):
         return {"packages": [{"local": "override-view-adapter-dep"}]}
 
-    def test_adapter_dependency(self, project, override_view_adapter_dep):
+    def test_adapter_dependency(self, project, override_view_adapter_dep, set_up_deprecations):
         run_dbt(["deps"])
         # this should error because the override is buggy
         run_dbt(["run"], expect_pass=False)
 
+        # overriding a built-in materialization scoped to adapter from package is deprecated
+        assert deprecations.active_deprecations == {"package-materialization-override"}
+
 
 class TestOverrideDefaultDependency:
     @pytest.fixture(scope="class")
     def packages(self):
         return {"packages": [{"local": "override-view-default-dep"}]}
 
-    def test_default_dependency(self, project, override_view_default_dep):
+    def test_default_dependency(self, project, override_view_default_dep, set_up_deprecations):
+        run_dbt(["deps"])
+        # this should error because the override is buggy
+        run_dbt(["run"], expect_pass=False)
+
+        # overriding a built-in materialization from package is deprecated
+        assert deprecations.active_deprecations == {"package-materialization-override"}
+
+
+root_view_override_macro = """
+{% materialization view, default %}
+ {{ return(view_default_override.materialization_view_default()) }}
+{% endmaterialization %}
+"""
+
+
+class TestOverrideDefaultDependencyRootOverride:
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {"packages": [{"local": "override-view-default-dep"}]}
+
+    @pytest.fixture(scope="class")
+    def macros(self):
+        return {"my_view.sql": root_view_override_macro}
+
+    def test_default_dependency_with_root_override(
+        self, project, override_view_default_dep, set_up_deprecations
+    ):
         run_dbt(["deps"])
         # this should error because the override is buggy
         run_dbt(["run"], expect_pass=False)
 
+        # using an package-overriden built-in materialization in a root matereialization is _not_ deprecated
+        assert deprecations.active_deprecations == set()
+
+
+class TestCustomMaterializationDependency:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"model.sql": models_custom_materialization__model_sql}
+
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {"packages": [{"local": "custom-materialization-dep"}]}
+
+    def test_custom_materialization_deopendency(
+        self, project, custom_materialization_dep, set_up_deprecations
+    ):
+        run_dbt(["deps"])
+        # custom materilization is valid
+        run_dbt(["run"])
+
+        # using a custom materialization is from an installed package is _not_ deprecated
+        assert deprecations.active_deprecations == set()
+
 
 class TestOverrideAdapterDependencyPassing:
     @pytest.fixture(scope="class")

tests/unit/test_events.py

@@ -138,6 +138,9 @@ def test_event_codes(self):
     types.ConfigLogPathDeprecation(deprecated_path=""),
     types.ConfigTargetPathDeprecation(deprecated_path=""),
     types.CollectFreshnessReturnSignature(),
+    types.PackageMaterializationOverrideDeprecation(
+        package_name="my_package", materialization_name="view"
+    ),
     # E - DB Adapter ======================
     types.AdapterEventDebug(),
     types.AdapterEventInfo(),