Merge pull request #1025 from fishtown-analytics/feature/cache-relations

Feature/cache relations (#911)

dbt/adapters/bigquery/impl.py

@@ -41,12 +41,12 @@ class BigQueryAdapter(PostgresAdapter):
  "expand_target_column_types",
  "load_dataframe",
  "get_missing_columns",
+ "cache_new_relation",
 
  "create_schema",
  "alter_table_add_columns",
 
  # versions of adapter functions that take / return Relations
- "list_relations",
  "get_relation",
  "drop_relation",
  "rename_relation",
@@ -180,7 +180,10 @@ def close(cls, connection):
 
  return connection
 
- def list_relations(self, schema, model_name=None):
+ def _link_cached_relations(self, manifest, schemas):
+ pass
+
+ def _list_relations(self, schema, model_name=None):
  connection = self.get_connection(model_name)
  client = connection.handle
 
@@ -201,27 +204,27 @@ def list_relations(self, schema, model_name=None):
  # This will 404 if the dataset does not exist. This behavior mirrors
  # the implementation of list_relations for other adapters
  try:
- return [self.bq_table_to_relation(table) for table in all_tables]
+ return [self._bq_table_to_relation(table) for table in all_tables]
  except google.api_core.exceptions.NotFound as e:
  return []
 
- def get_relation(self, schema=None, identifier=None,
-  relations_list=None, model_name=None):
- if schema is None and relations_list is None:
- raise dbt.exceptions.RuntimeException(
-  'get_relation needs either a schema to query, or a list '
- 'of relations to use')
-
- if relations_list is None and identifier is not None:
- table = self.get_bq_table(schema, identifier)
+ def get_relation(self, schema, identifier, model_name=None):
+ if self._schema_is_cached(schema, model_name):
+  # if it's in the cache, use the parent's model of going through
+ # the relations cache and picking out the relation
+ return super(BigQueryAdapter, self).get_relation(
+ schema=schema,
+ identifier=identifier,
+  model_name=model_name
+ )
 
- return self.bq_table_to_relation(table)
-
- return super(BigQueryAdapter, self).get_relation(
- schema, identifier, relations_list,
- model_name)
+ table = self._get_bq_table(schema, identifier)
+ return self._bq_table_to_relation(table)
 
  def drop_relation(self, relation, model_name=None):
+ if self._schema_is_cached(relation.schema, model_name):
+ self.cache.drop(relation)
+
  conn = self.get_connection(model_name)
  client = conn.handle
 
@@ -515,7 +518,7 @@ def get_dataset(self, dataset_name, model_name=None):
  dataset_ref = conn.handle.dataset(dataset_name)
  return google.cloud.bigquery.Dataset(dataset_ref)
 
- def bq_table_to_relation(self, bq_table):
+ def _bq_table_to_relation(self, bq_table):
  if bq_table is None:
  return None
 
@@ -529,7 +532,7 @@ def bq_table_to_relation(self, bq_table):
  },
  type=self.RELATION_TYPES.get(bq_table.table_type))
 
- def get_bq_table(self, dataset_name, identifier, model_name=None):
+ def _get_bq_table(self, dataset_name, identifier, model_name=None):
  conn = self.get_connection(model_name)
 
  dataset = self.get_dataset(dataset_name, model_name)