Make databases optional

CHANGELOG.md

@@ -13,6 +13,9 @@
 - Substantial performance improvements for parsing on large projects, especially projects with many docs definition. ([#2480](https://github.com/fishtown-analytics/dbt/issues/2480), [#2481](https://github.com/fishtown-analytics/dbt/pull/2481))
 - Expose Snowflake query id in case of an exception raised by connector ([#2201](https://github.com/fishtown-analytics/dbt/issues/2201), [#2358](https://github.com/fishtown-analytics/dbt/pull/2358))
 
+### Under the hood
+- Better support for optional database fields in adapters ([#2487](https://github.com/fishtown-analytics/dbt/issues/2487) [#2489](https://github.com/fishtown-analytics/dbt/pull/2489))
+
 Contributors:
 - [@dmateusp](https://github.com/dmateusp) ([#2475](https://github.com/fishtown-analytics/dbt/pull/2475))
 - [@ChristianKohlberg](https://github.com/ChristianKohlberg) (#2358](https://github.com/fishtown-analytics/dbt/pull/2358))

core/dbt/adapters/base/impl.py

@@ -264,7 +264,7 @@ def load_internal_manifest(self) -> Manifest:
  ###
  # Caching methods
  ###
- def _schema_is_cached(self, database: str, schema: str) -> bool:
+ def _schema_is_cached(self, database: Optional[str], schema: str) -> bool:
  """Check if the schema is cached, and by default logs if it is not."""
 
  if dbt.flags.USE_CACHE is False:
@@ -341,12 +341,8 @@ def _relations_cache_for_schemas(self, manifest: Manifest) -> None:
  # it's possible that there were no relations in some schemas. We want
  # to insert the schemas we query into the cache's `.schemas` attribute
  # so we can check it later
- cache_update: Set[Tuple[str, Optional[str]]] = set()
+ cache_update: Set[Tuple[Optional[str], Optional[str]]] = set()
  for relation in cache_schemas:
- if relation.database is None:
- raise InternalException(
- 'Got a None database in a cached schema!'
- )
  cache_update.add((relation.database, relation.schema))
  self.cache.update_schemas(cache_update)
 
@@ -646,7 +642,9 @@ def expand_target_column_types(
 
  self.expand_column_types(from_relation, to_relation)
 
- def list_relations(self, database: str, schema: str) -> List[BaseRelation]:
+ def list_relations(
+ self, database: Optional[str], schema: str
+ ) -> List[BaseRelation]:
  if self._schema_is_cached(database, schema):
  return self.cache.get_relations(database, schema)
 

core/dbt/adapters/base/relation.py

@@ -6,7 +6,7 @@
 from collections.abc import Mapping, Hashable
 from dataclasses import dataclass, fields
 from typing import (
- Optional, TypeVar, Generic, Any, Type, Dict, Union, List, Iterator, Tuple,
+ Optional, TypeVar, Generic, Any, Type, Dict, Union, Iterator, Tuple,
  Set
 )
 from typing_extensions import Protocol
@@ -278,16 +278,11 @@ def _render_iterator(
  yield key, path_part
 
  def render(self) -> str:
- parts: List[str] = [
- part for _, part in self._render_iterator() if part is not None
- ]
-
- if len(parts) == 0:
- raise dbt.exceptions.RuntimeException(
- "No path parts are included! Nothing to render."
- )
-
- return '.'.join(parts)
+ # if there is nothing set, this will return the empty string.
+ return '.'.join(
+ part for _, part in self._render_iterator()
+ if part is not None
+ )
 
  def quoted(self, identifier):
  return '{quote_char}{identifier}{quote_char}'.format(

core/dbt/adapters/cache.py

@@ -4,25 +4,20 @@
 import threading
 
 from dbt.logger import CACHE_LOGGER as logger
+from dbt.utils import lowercase
 import dbt.exceptions
 
 _ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier')
 
 
-def _lower(value: Optional[str]) -> Optional[str]:
- """Postgres schemas can be None so we can't just call lower()."""
- if value is None:
- return None
- return value.lower()
-
-
 def _make_key(relation) -> _ReferenceKey:
  """Make _ReferenceKeys with lowercase values for the cache so we don't have
  to keep track of quoting
  """
- return _ReferenceKey(_lower(relation.database),
- _lower(relation.schema),
- _lower(relation.identifier))
+ # databases and schemas can both be None
+ return _ReferenceKey(lowercase(relation.database),
+ lowercase(relation.schema),
+ lowercase(relation.identifier))
 
 
 def dot_separated(key: _ReferenceKey) -> str:
@@ -53,15 +48,15 @@ def __str__(self) -> str:
 
  @property
  def database(self) -> Optional[str]:
- return _lower(self.inner.database)
+ return lowercase(self.inner.database)
 
  @property
  def schema(self) -> Optional[str]:
- return _lower(self.inner.schema)
+ return lowercase(self.inner.schema)
 
  @property
  def identifier(self) -> Optional[str]:
- return _lower(self.inner.identifier)
+ return lowercase(self.inner.identifier)
 
  def __copy__(self):
  new = self.__class__(self.inner)
@@ -190,7 +185,7 @@ def add_schema(
  :param database: The database name to add.
  :param schema: The schema name to add.
  """
- self.schemas.add((_lower(database), _lower(schema)))
+ self.schemas.add((lowercase(database), lowercase(schema)))
 
  def drop_schema(
  self, database: Optional[str], schema: Optional[str],
@@ -199,7 +194,7 @@ def drop_schema(
 
  Then remove all its contents (and their dependents, etc) as well.
  """
- key = (_lower(database), _lower(schema))
+ key = (lowercase(database), lowercase(schema))
  if key not in self.schemas:
  return
 
@@ -217,7 +212,7 @@ def update_schemas(self, schemas: Iterable[Tuple[Optional[str], str]]):
 
  :param schemas: An iterable of the schema names to add.
  """
- self.schemas.update((_lower(d), s.lower()) for (d, s) in schemas)
+ self.schemas.update((lowercase(d), s.lower()) for (d, s) in schemas)
 
  def __contains__(self, schema_id: Tuple[Optional[str], str]):
  """A schema is 'in' the relations cache if it is in the set of cached
@@ -226,7 +221,7 @@ def __contains__(self, schema_id: Tuple[Optional[str], str]):
  :param schema_id: The db name and schema name to look up.
  """
  db, schema = schema_id
- return (_lower(db), schema.lower()) in self.schemas
+ return (lowercase(db), schema.lower()) in self.schemas
 
  def dump_graph(self):
  """Dump a key-only representation of the schema to a dictionary. Every
@@ -484,13 +479,13 @@ def get_relations(
  :return List[BaseRelation]: The list of relations with the given
  schema
  """
- database = _lower(database)
- schema = _lower(schema)
+ database = lowercase(database)
+ schema = lowercase(schema)
  with self.lock:
  results = [
  r.inner for r in self.relations.values()
- if (_lower(r.schema) == schema and
- _lower(r.database) == database)
+ if (lowercase(r.schema) == schema and
+ lowercase(r.database) == database)
  ]
 
  if None in results:
@@ -509,7 +504,7 @@ def _list_relations_in_schema(
  self, database: Optional[str], schema: Optional[str]
  ) -> List[_CachedRelation]:
  """Get the relations in a schema. Callers should hold the lock."""
- key = (_lower(database), _lower(schema))
+ key = (lowercase(database), lowercase(schema))
 
  to_remove: List[_CachedRelation] = []
  for cachekey, relation in self.relations.items():

core/dbt/contracts/graph/parsed.py

@@ -83,7 +83,7 @@ def add_node(self, value: str):
 
 @dataclass
 class HasRelationMetadata(JsonSchemaMixin, Replaceable):
- database: str
+ database: Optional[str]
  schema: str
 
 

core/dbt/contracts/results.py

@@ -10,6 +10,7 @@
  JsonOnly,
  GLOBAL_LOGGER as logger,
 )
+from dbt.utils import lowercase
 from hologram.helpers import StrEnum
 from hologram import JsonSchemaMixin
 
@@ -238,7 +239,7 @@ class FreshnessRunOutput(JsonSchemaMixin, Writable):
 
 CatalogKey = NamedTuple(
  'CatalogKey',
- [('database', str), ('schema', str), ('name', str)]
+ [('database', Optional[str]), ('schema', str), ('name', str)]
 )
 
 
@@ -268,7 +269,7 @@ class ColumnMetadata(JsonSchemaMixin):
 @dataclass
 class TableMetadata(JsonSchemaMixin):
  type: str
- database: str
+ database: Optional[str]
  schema: str
  name: str
  comment: Optional[str]
@@ -285,7 +286,7 @@ class CatalogTable(JsonSchemaMixin, Replaceable):
 
  def key(self) -> CatalogKey:
  return CatalogKey(
- self.metadata.database.lower(),
+ lowercase(self.metadata.database),
  self.metadata.schema.lower(),
  self.metadata.name.lower(),
  )

core/dbt/include/global_project/macros/etc/get_custom_database.sql

@@ -14,6 +14,10 @@
 
 #}
 {% macro generate_database_name(custom_database_name=none, node=none) -%}
+ {% do return(adapter_macro('generate_database_name', custom_database_name, node)) %}
+{%- endmacro %}
+
+{% macro default__generate_database_name(custom_database_name=none, node=none) -%}
  {%- set default_database = target.database -%}
  {%- if custom_database_name is none -%}
 

core/dbt/node_runners.py

@@ -27,6 +27,7 @@
 import dbt.tracking
 import dbt.ui.printer
 import dbt.flags
+import dbt.utils
 
 
 INTERNAL_ERROR_STRING = """This is an error in dbt. Please try again. If \
@@ -491,8 +492,7 @@ def _build_run_result(self, node, start_time, error, status, timing_info,
  skip=False, failed=None):
  execution_time = time.time() - start_time
  thread_id = threading.current_thread().name
- if status is not None:
- status = status.lower()
+ status = dbt.utils.lowercase(status)
  return PartialResult(
  node=node,
  status=status,

core/dbt/parser/base.py

@@ -120,7 +120,9 @@ def __call__(
  self, parsed_node: Any, config_dict: Dict[str, Any]
  ) -> None:
  override = config_dict.get(self.component)
- new_value = self.updater(override, parsed_node).strip()
+ new_value = self.updater(override, parsed_node)
+ if isinstance(new_value, str):
+ new_value = new_value.strip()
  setattr(parsed_node, self.component, new_value)
 
 

core/dbt/task/generate.py

@@ -62,9 +62,15 @@ def __init__(self, columns: List[PrimitiveDict]):
  self.add_column(col)
 
  def get_table(self, data: PrimitiveDict) -> CatalogTable:
+ database = data.get('table_database')
+ if database is None:
+ dkey: Optional[str] = None
+ else:
+ dkey = str(database)
+
  try:
  key = CatalogKey(
- str(data['table_database']),
+ dkey,
  str(data['table_schema']),
  str(data['table_name']),
  )
@@ -164,8 +170,9 @@ def format_stats(stats: PrimitiveDict) -> StatsDict:
 
 
 def mapping_key(node: CompileResultNode) -> CatalogKey:
+ dkey = dbt.utils.lowercase(node.database)
  return CatalogKey(
- node.database.lower(), node.schema.lower(), node.identifier.lower()
+ dkey, node.schema.lower(), node.identifier.lower()
  )
 
 

core/dbt/task/runnable.py

@@ -45,12 +45,6 @@
 RUNNING_STATE = DbtProcessState('running')
 
 
-def _lower(value: Optional[str]) -> Optional[str]:
- if value is None:
- return value
- return value.lower()
-
-
 def write_manifest(config, manifest):
  if dbt.flags.WRITE_JSON:
  manifest.write(os.path.join(config.target_path, MANIFEST_FILE_NAME))
@@ -453,28 +447,32 @@ def create_schemas(self, adapter, selected_uids: Iterable[str]):
  )
  required_databases.add(db_only)
 
- existing_schemas_lowered: Set[Tuple[str, Optional[str]]] = set()
+ existing_schemas_lowered: Set[Tuple[Optional[str], Optional[str]]]
+ existing_schemas_lowered = set()
 
- def list_schemas(db_only: BaseRelation) -> List[Tuple[str, str]]:
- # the database name should never be None here (or where are we
- # listing schemas from?)
+ def list_schemas(
+ db_only: BaseRelation
+ ) -> List[Tuple[Optional[str], str]]:
+ # the database can be None on some warehouses that don't support it
+ database_quoted: Optional[str]
+ db_lowercase = dbt.utils.lowercase(db_only.database)
  if db_only.database is None:
- raise InternalException(
-  f'Got an invalid database-only portion of {db_only} '
-  f'(database was None)'
- )
- database_name: str = db_only.database
- database_quoted = str(db_only)
- with adapter.connection_named(f'list_{database_name}'):
+ database_quoted = None
+ conn_name = 'list_schemas'
+ else:
+ database_quoted = str(db_only)
+  conn_name = f'list_{db_only.database}'
+
+ with adapter.connection_named(conn_name):
  # we should never create a null schema, so just filter them out
  return [
- (database_name.lower(), s.lower())
+ (db_lowercase, s.lower())
  for s in adapter.list_schemas(database_quoted)
  if s is not None
  ]
 
  def create_schema(relation: BaseRelation) -> None:
- db = relation.database
+ db = relation.database or ''
  schema = relation.schema
  with adapter.connection_named(f'create_{db}_{schema}'):
  adapter.create_schema(relation)
@@ -491,18 +489,15 @@ def create_schema(relation: BaseRelation) -> None:
  existing_schemas_lowered.update(ls_future.result())
 
  for info in required_schemas:
- if info.database is None:
- raise InternalException(
- 'Got an information schema with no database!'
- )
  if info.schema is None:
  # we are not in the business of creating null schemas, so
  # skip this
  continue
- db: str = info.database
+ db: Optional[str] = info.database
+ db_lower: Optional[str] = dbt.utils.lowercase(db)
  schema: str = info.schema
 
- db_schema = (db.lower(), schema.lower())
+ db_schema = (db_lower, schema.lower())
  if db_schema not in existing_schemas_lowered:
  existing_schemas_lowered.add(db_schema)
  create_futures.append(

core/dbt/utils.py

@@ -490,6 +490,13 @@ def coerce_dict_str(value: Any) -> Optional[Dict[str, Any]]:
  return None
 
 
+def lowercase(value: Optional[str]) -> Optional[str]:
+ if value is None:
+ return None
+ else:
+ return value.lower()
+
+
 # some types need to make constants available to the jinja context as
 # attributes, and regular properties only work with objects. maybe this should
 # be handled by the RelationProxy?