Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update materialized views to use RelationConfigs and remove refs to dbt.utils #9291

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
2474722
moving types_pb2.py to common/events
colin-rogers-dbt Oct 25, 2023
07743b7
merge
colin-rogers-dbt Oct 25, 2023
252e3e3
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Oct 26, 2023
ff9d519
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Nov 2, 2023
bf079b1
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Nov 28, 2023
3187ded
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Nov 28, 2023
4da67bf
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 5, 2023
c03823f
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 6, 2023
2811ccd
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 7, 2023
e88138a
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 7, 2023
977a842
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 7, 2023
3ddfcdd
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 12, 2023
082b0c6
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 14, 2023
611981f
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 14, 2023
3665fb1
Merge branch 'feature/decouple-adapters-from-core' of https://github.…
colin-rogers-dbt Dec 14, 2023
aeb64a2
add AdapterRuntimeConfig protocol and clean up dbt-postgress core imp…
colin-rogers-dbt Dec 14, 2023
566abc3
add changie
colin-rogers-dbt Dec 14, 2023
97172c3
remove AdapterRuntimeConfig
colin-rogers-dbt Dec 15, 2023
98b1280
update changelog
colin-rogers-dbt Dec 15, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Under the Hood-20231214-122134.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: have dbt-postgres use RelationConfig protocol for materialized views'
time: 2023-12-14T12:21:34.756973-08:00
custom:
Author: colin-rogers-dbt
Issue: "9292"
4 changes: 2 additions & 2 deletions plugins/postgres/dbt/adapters/postgres/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
IndexConfigError,
UnexpectedDbReferenceError,
)
import dbt.utils
from dbt.common.utils import encoding as dbt_encoding


GET_RELATIONS_MACRO_NAME = "postgres__get_relations"
Expand All @@ -38,7 +38,7 @@ def render(self, relation):
now = datetime.utcnow().isoformat()
inputs = self.columns + [relation.render(), str(self.unique), str(self.type), now]
string = "_".join(inputs)
return dbt.utils.md5(string)
return dbt_encoding.md5(string)

@classmethod
def parse(cls, raw_index) -> Optional["PostgresIndexConfig"]:
Expand Down
9 changes: 3 additions & 6 deletions plugins/postgres/dbt/adapters/postgres/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
RelationConfigChangeAction,
RelationResults,
)
from dbt.context.providers import RuntimeConfigObject
from dbt.adapters.contracts.relation import RelationType
from dbt.adapters.contracts.relation import RelationType, RelationConfig
from dbt.common.exceptions import DbtRuntimeError

from dbt.adapters.postgres.relation_configs import (
Expand Down Expand Up @@ -52,16 +51,14 @@ def relation_max_name_length(self):
return MAX_CHARACTERS_IN_IDENTIFIER

def get_materialized_view_config_change_collection(
self, relation_results: RelationResults, runtime_config: RuntimeConfigObject
self, relation_results: RelationResults, relation_config: RelationConfig
) -> Optional[PostgresMaterializedViewConfigChangeCollection]:
config_change_collection = PostgresMaterializedViewConfigChangeCollection()

existing_materialized_view = PostgresMaterializedViewConfig.from_relation_results(
relation_results
)
new_materialized_view = PostgresMaterializedViewConfig.from_model_node(
runtime_config.model
)
new_materialized_view = PostgresMaterializedViewConfig.from_config(relation_config)

config_change_collection.indexes = self._get_index_config_changes(
existing_materialized_view.indexes, new_materialized_view.indexes
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from dataclasses import dataclass, field
from typing import Set, FrozenSet, List
from typing import Set, FrozenSet, List, Dict
from typing_extensions import Self

import agate
from dbt.adapters.relation_configs import (
Expand All @@ -8,7 +9,7 @@
RelationConfigValidationMixin,
RelationConfigValidationRule,
)
from dbt.contracts.graph.nodes import ModelNode
from dbt.adapters.contracts.relation import RelationConfig
from dbt.common.exceptions import DbtRuntimeError

from dbt.adapters.postgres.relation_configs.constants import MAX_CHARACTERS_IN_IDENTIFIER
Expand Down Expand Up @@ -54,37 +55,35 @@ def validation_rules(self) -> Set[RelationConfigValidationRule]:
}

@classmethod
def from_dict(cls, config_dict: dict) -> "PostgresMaterializedViewConfig":
def from_dict(cls, config_dict: dict) -> Self:
kwargs_dict = {
"table_name": config_dict.get("table_name"),
"query": config_dict.get("query"),
"indexes": frozenset(
PostgresIndexConfig.from_dict(index) for index in config_dict.get("indexes", {})
),
}
materialized_view: "PostgresMaterializedViewConfig" = super().from_dict(kwargs_dict) # type: ignore
materialized_view: Self = super().from_dict(kwargs_dict) # type: ignore
return materialized_view

@classmethod
def from_model_node(cls, model_node: ModelNode) -> "PostgresMaterializedViewConfig":
materialized_view_config = cls.parse_model_node(model_node)
def from_config(cls, relation_config: RelationConfig) -> Self:
materialized_view_config = cls.parse_config(relation_config)
materialized_view = cls.from_dict(materialized_view_config)
return materialized_view

@classmethod
def parse_model_node(cls, model_node: ModelNode) -> dict:
indexes: List[dict] = model_node.config.extra.get("indexes", [])
def parse_config(cls, relation_config: RelationConfig) -> Dict:
indexes: List[dict] = relation_config.config.extra.get("indexes", [])
config_dict = {
"table_name": model_node.identifier,
"query": model_node.compiled_code,
"table_name": relation_config.identifier,
"query": relation_config.compiled_code,
"indexes": [PostgresIndexConfig.parse_model_node(index) for index in indexes],
}
return config_dict

@classmethod
def from_relation_results(
cls, relation_results: RelationResults
) -> "PostgresMaterializedViewConfig":
def from_relation_results(cls, relation_results: RelationResults) -> Self:
materialized_view_config = cls.parse_relation_results(relation_results)
materialized_view = cls.from_dict(materialized_view_config)
return materialized_view
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,6 @@

{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %}
{% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %}
{% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config) %}
{% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config.model) %}
{% do return(_configuration_changes) %}
{% endmacro %}
Loading