Skip to content

Commit

Permalink
Alias database to allow matching between rows and Manifest
Browse files Browse the repository at this point in the history
Currently the docs generation is broken because we need to supply
the database name when fetching the relations:

if dct['table_database'] is None:
    dct['table_database'] = dct['table_schema']

However, when we get the manifest we don't get the database:

{CatalogKey(database='', schema='fokko', name='logistical_configuration_data'):
	['model.dbtlake.logistical_configuration_data']}

Therefore the keys never line up, and we can't match the Catalogs:

https://github.com/fishtown-analytics/dbt/blob/9d0eab630511723cd0bc328f6f11d3ffe6c8f879/core/dbt/task/generate.py#L108

We get from the describe relations:

CatalogKey(database='fokko', schema='fokko', name='logistical_configuration_data')

Due to the logic above.
  • Loading branch information
Fokko committed May 22, 2020
1 parent 8a2b24e commit cabc03b
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 7 deletions.
3 changes: 3 additions & 0 deletions dbt/adapters/spark/connections.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ class SparkCredentials(Credentials):
organization: str = '0'
connect_retries: int = 0
connect_timeout: int = 10
_ALIASES = {
'database': 'schema'
}

def __post_init__(self):
# spark classifies database and schema as the same thing
Expand Down
13 changes: 6 additions & 7 deletions dbt/adapters/spark/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import agate
import dbt.exceptions
import dbt
from dbt.adapters.base.relation import SchemaSearchMap
from dbt.adapters.base.relation import SchemaSearchMap, RelationType
from dbt.adapters.sql import SQLAdapter
from dbt.node_types import NodeType

Expand Down Expand Up @@ -117,7 +117,8 @@ def list_relations_without_caching(

relations = []
for _schema, name, _, information in results:
rel_type = ('view' if 'Type: VIEW' in information else 'table')
rel_type = (RelationType.View
if 'Type: VIEW' in information else RelationType.Table)
relation = self.Relation.create(
schema=_schema,
identifier=name,
Expand All @@ -132,7 +133,6 @@ def get_relation(
) -> Optional[BaseRelation]:
if not self.Relation.include_policy.database:
database = None

return super().get_relation(database, schema, identifier)

def parse_describe_extended(
Expand Down Expand Up @@ -256,9 +256,8 @@ def _parse_relation(relation: Relation,

return columns

def _massage_column_for_catalog(
self, column: SparkColumn
) -> Dict[str, Any]:
@staticmethod
def _massage_column_for_catalog(column: SparkColumn) -> Dict[str, Any]:
dct = column.to_dict()
# different expectations here - Column.column is the name
dct['column_name'] = dct.pop('column')
Expand All @@ -279,7 +278,7 @@ def _get_catalog_for_relations(self, database: str, schema: str):
)
return agate.Table.from_object(columns)

def _get_cache_schemas(self, manifest, exec_only=False):
def _get_cache_schemas(self, manifest, exec_only=False) -> SchemaSearchMap:
info_schema_name_map = SchemaSearchMap()
for node in manifest.nodes.values():
if exec_only and node.resource_type not in NodeType.executable():
Expand Down

0 comments on commit cabc03b

Please sign in to comment.