Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize list_relations_without_caching() macro to avoid calling get_relationship() on every object. #181

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions dbt/adapters/impala/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,25 +113,25 @@ def list_relations_without_caching(

relations = []
for row in results:
if len(row) != 2:
if len(row) != 1:
raise dbt.exceptions.DbtRuntimeError(
f'Invalid value from "show table extended ...", '
f"got {len(row)} values, expected 4"
f'Invalid value from "show tables in ...", '
f"got {len(row)} values, expected 1"
)
_identifier = row[0]
_rel_type = row[1]

relation = self.Relation.create(
database=None,
schema=schema_relation.schema,
identifier=_identifier,
type=_rel_type,
information=_identifier,
adapter=self
)
relations.append(relation)

return relations


def get_columns_in_relation(self, relation: Relation) -> List[ImpalaColumn]:
cached_relations = self.cache.get_relations(relation.database, relation.schema)
cached_relation = next(
Expand Down
25 changes: 25 additions & 0 deletions dbt/adapters/impala/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,17 @@
# limitations under the License.

from dataclasses import dataclass, field
from typing import Optional

from dbt.adapters.base.relation import BaseRelation, Policy

import dbt.adapters.impala.cloudera_tracking as tracker

from dbt.contracts.relation import (
RelationType
)

GET_RELATION_TYPE_MACRO_NAME = "get_relation_type"

@dataclass
class ImpalaQuotePolicy(Policy):
Expand All @@ -39,6 +45,7 @@ class ImpalaRelation(BaseRelation):
include_policy: ImpalaIncludePolicy = field(default_factory=lambda: ImpalaIncludePolicy())
quote_character: str = None
information: str = None
adapter: str = None

def __post_init__(self):
if self.type:
Expand All @@ -51,6 +58,24 @@ def __post_init__(self):
}
)

def __getattribute__(self, name):
if name == 'type':
type_info = object.__getattribute__(self, name)
if type_info:
return type_info
elif self.adapter:
result = self.adapter.execute_macro(GET_RELATION_TYPE_MACRO_NAME,
kwargs= {"relation": ImpalaRelation.create(
database=self.database,
schema=self.schema,
identifier=self.identifier,
information=self.identifier,
)})
print("result is:", result)
return result
return None
return object.__getattribute__(self, name)

def render(self):
return super().render()

Expand Down
9 changes: 1 addition & 8 deletions dbt/include/impala/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -132,14 +132,7 @@

{%- for rs in result_set -%}
{% set obj_type = [] %}

{% do obj_type.append(rs[0]) %}

{% set obj_rel = relation.new_copy(relation.schema, rs[0]) %}
{% set rel_type = get_relation_type(obj_rel) %}
{% do obj_type.append(rel_type) %}

{% do objects_with_type.append(obj_type) %}
{% do objects_with_type.append([rs[0]]) %}
{%- endfor -%}

{{ return(objects_with_type) }}
Expand Down