Skip to content

Commit 1036b82

Browse files
alixhamitswast
alixhami
authored and
tswast
committed
BigQuery: replaces table.create() with client.create_table() (#4038)
* adds client.create_table() * removes table.create() * passes system tests * fixes rebase conflicts * fixes coverage
1 parent ce9a06c commit 1036b82

File tree

5 files changed

+205
-273
lines changed

5 files changed

+205
-273
lines changed

‎bigquery/google/cloud/bigquery/client.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,24 @@ def create_dataset(self, dataset):
190190
method='POST', path=path, data=dataset._build_resource())
191191
return Dataset.from_api_repr(api_response)
192192

193+
def create_table(self, table):
194+
"""API call: create a table via a PUT request
195+
196+
See
197+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/insert
198+
199+
:type table: :class:`~google.cloud.bigquery.table.Table`
200+
:param table: A ``Table`` populated with the desired initial state.
201+
202+
:rtype: ":class:`~google.cloud.bigquery.table.Table`"
203+
:returns: a new ``Table`` returned from the service.
204+
"""
205+
path = '/projects/%s/datasets/%s/tables' % (
206+
table.project, table.dataset_id)
207+
api_response = self._connection.api_request(
208+
method='POST', path=path, data=table._build_resource())
209+
return Table.from_api_repr(api_response, self)
210+
193211
def get_dataset(self, dataset_ref):
194212
"""Fetch the dataset referenced by ``dataset_ref``
195213

‎bigquery/google/cloud/bigquery/table.py‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -653,24 +653,6 @@ def _build_resource(self):
653653

654654
return resource
655655

656-
def create(self, client=None):
657-
"""API call: create the table via a PUT request
658-
659-
See
660-
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/insert
661-
662-
:type client: :class:`~google.cloud.bigquery.client.Client` or
663-
``NoneType``
664-
:param client: the client to use. If not passed, falls back to the
665-
``client`` stored on the current dataset.
666-
"""
667-
client = self._require_client(client)
668-
path = '/projects/%s/datasets/%s/tables' % (
669-
self._project, self._dataset_id)
670-
api_response = client._connection.api_request(
671-
method='POST', path=path, data=self._build_resource())
672-
self._set_properties(api_response)
673-
674656
def exists(self, client=None):
675657
"""API call: test for the existence of the table via a GET request
676658

‎bigquery/tests/system.py‎

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -180,18 +180,19 @@ def test_list_datasets(self):
180180

181181
def test_create_table(self):
182182
dataset = self.temp_dataset(_make_dataset_id('create_table'))
183-
184-
TABLE_NAME = 'test_table'
183+
table_id = 'test_table'
185184
full_name = bigquery.SchemaField('full_name', 'STRING',
186185
mode='REQUIRED')
187186
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
188-
table = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
189-
client=Config.CLIENT)
190-
self.assertFalse(table.exists())
191-
table.create()
187+
table_arg = Table(dataset.table(table_id), schema=[full_name, age],
188+
client=Config.CLIENT)
189+
self.assertFalse(table_arg.exists())
190+
191+
table = retry_403(Config.CLIENT.create_table)(table_arg)
192192
self.to_delete.insert(0, table)
193+
193194
self.assertTrue(table.exists())
194-
self.assertEqual(table.table_id, TABLE_NAME)
195+
self.assertEqual(table.table_id, table_id)
195196

196197
def test_get_table_w_public_dataset(self):
197198
PUBLIC = 'bigquery-public-data'
@@ -227,10 +228,10 @@ def test_list_dataset_tables(self):
227228
mode='REQUIRED')
228229
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
229230
for table_name in tables_to_create:
230-
created_table = Table(dataset.table(table_name),
231-
schema=[full_name, age],
232-
client=Config.CLIENT)
233-
created_table.create()
231+
table = Table(dataset.table(table_name),
232+
schema=[full_name, age],
233+
client=Config.CLIENT)
234+
created_table = retry_403(Config.CLIENT.create_table)(table)
234235
self.to_delete.insert(0, created_table)
235236

236237
# Retrieve the tables.
@@ -249,10 +250,10 @@ def test_patch_table(self):
249250
full_name = bigquery.SchemaField('full_name', 'STRING',
250251
mode='REQUIRED')
251252
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
252-
table = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
253-
client=Config.CLIENT)
254-
self.assertFalse(table.exists())
255-
table.create()
253+
table_arg = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
254+
client=Config.CLIENT)
255+
self.assertFalse(table_arg.exists())
256+
table = retry_403(Config.CLIENT.create_table)(table_arg)
256257
self.to_delete.insert(0, table)
257258
self.assertTrue(table.exists())
258259
self.assertIsNone(table.friendly_name)
@@ -268,10 +269,10 @@ def test_update_table(self):
268269
full_name = bigquery.SchemaField('full_name', 'STRING',
269270
mode='REQUIRED')
270271
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
271-
table = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
272-
client=Config.CLIENT)
273-
self.assertFalse(table.exists())
274-
table.create()
272+
table_arg = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
273+
client=Config.CLIENT)
274+
self.assertFalse(table_arg.exists())
275+
table = retry_403(Config.CLIENT.create_table)(table_arg)
275276
self.to_delete.insert(0, table)
276277
self.assertTrue(table.exists())
277278
voter = bigquery.SchemaField('voter', 'BOOLEAN', mode='NULLABLE')
@@ -309,10 +310,10 @@ def test_insert_data_then_dump_table(self):
309310
mode='REQUIRED')
310311
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
311312
now = bigquery.SchemaField('now', 'TIMESTAMP')
312-
table = Table(dataset.table(TABLE_NAME), schema=[full_name, age, now],
313-
client=Config.CLIENT)
314-
self.assertFalse(table.exists())
315-
table.create()
313+
table_arg = Table(dataset.table(TABLE_NAME),
314+
schema=[full_name, age, now], client=Config.CLIENT)
315+
self.assertFalse(table_arg.exists())
316+
table = retry_403(Config.CLIENT.create_table)(table_arg)
316317
self.to_delete.insert(0, table)
317318
self.assertTrue(table.exists())
318319

@@ -346,9 +347,9 @@ def test_load_table_from_local_file_then_dump_table(self):
346347
full_name = bigquery.SchemaField('full_name', 'STRING',
347348
mode='REQUIRED')
348349
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
349-
table = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
350-
client=Config.CLIENT)
351-
table.create()
350+
table_arg = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
351+
client=Config.CLIENT)
352+
table = retry_403(Config.CLIENT.create_table)(table_arg)
352353
self.to_delete.insert(0, table)
353354

354355
with _NamedTemporaryFile() as temp:
@@ -450,9 +451,9 @@ def test_load_table_from_storage_then_dump_table(self):
450451
full_name = bigquery.SchemaField('full_name', 'STRING',
451452
mode='REQUIRED')
452453
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
453-
table = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
454-
client=Config.CLIENT)
455-
table.create()
454+
table_arg = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
455+
client=Config.CLIENT)
456+
table = retry_403(Config.CLIENT.create_table)(table_arg)
456457
self.to_delete.insert(0, table)
457458

458459
job = Config.CLIENT.load_table_from_storage(
@@ -652,9 +653,9 @@ def test_job_cancel(self):
652653
full_name = bigquery.SchemaField('full_name', 'STRING',
653654
mode='REQUIRED')
654655
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
655-
table = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
656-
client=Config.CLIENT)
657-
table.create()
656+
table_arg = Table(dataset.table(TABLE_NAME), schema=[full_name, age],
657+
client=Config.CLIENT)
658+
table = retry_403(Config.CLIENT.create_table)(table_arg)
658659
self.to_delete.insert(0, table)
659660

660661
job = Config.CLIENT.run_async_query(JOB_NAME, QUERY)
@@ -839,9 +840,9 @@ def _load_table_for_dml(self, rows, dataset_id, table_id):
839840
dataset = self.temp_dataset(dataset_id)
840841
greeting = bigquery.SchemaField(
841842
'greeting', 'STRING', mode='NULLABLE')
842-
table = Table(dataset.table(table_id), schema=[greeting],
843-
client=Config.CLIENT)
844-
table.create()
843+
table_arg = Table(dataset.table(table_id), schema=[greeting],
844+
client=Config.CLIENT)
845+
table = retry_403(Config.CLIENT.create_table)(table_arg)
845846
self.to_delete.insert(0, table)
846847

847848
with _NamedTemporaryFile() as temp:
@@ -1228,9 +1229,9 @@ def test_insert_nested_nested(self):
12281229
]
12291230
table_name = 'test_table'
12301231
dataset = self.temp_dataset(_make_dataset_id('issue_2951'))
1231-
table = Table(dataset.table(table_name), schema=schema,
1232-
client=Config.CLIENT)
1233-
table.create()
1232+
table_arg = Table(dataset.table(table_name), schema=schema,
1233+
client=Config.CLIENT)
1234+
table = retry_403(Config.CLIENT.create_table)(table_arg)
12341235
self.to_delete.insert(0, table)
12351236

12361237
table.insert_data(to_insert)
@@ -1245,9 +1246,9 @@ def test_create_table_insert_fetch_nested_schema(self):
12451246
dataset = self.temp_dataset(
12461247
_make_dataset_id('create_table_nested_schema'))
12471248
schema = _load_json_schema()
1248-
table = Table(dataset.table(table_name), schema=schema,
1249-
client=Config.CLIENT)
1250-
table.create()
1249+
table_arg = Table(dataset.table(table_name), schema=schema,
1250+
client=Config.CLIENT)
1251+
table = retry_403(Config.CLIENT.create_table)(table_arg)
12511252
self.to_delete.insert(0, table)
12521253
self.assertTrue(table.exists())
12531254
self.assertEqual(table.table_id, table_name)

‎bigquery/tests/unit/test_client.py‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,151 @@ def test_create_dataset_w_attrs(self):
399399
self.assertEqual(ds.default_table_expiration_ms, 3600)
400400
self.assertEqual(ds.labels, LABELS)
401401

402+
def test_create_table_w_day_partition(self):
403+
from google.cloud.bigquery.table import Table
404+
405+
project = 'PROJECT'
406+
dataset_id = 'dataset_id'
407+
table_id = 'table-id'
408+
path = 'projects/%s/datasets/%s/tables' % (
409+
project, dataset_id)
410+
creds = _make_credentials()
411+
client = self._make_one(project=project, credentials=creds)
412+
resource = {
413+
'id': '%s:%s:%s' % (project, dataset_id, table_id),
414+
'tableReference': {
415+
'projectId': project,
416+
'datasetId': dataset_id,
417+
'tableId': table_id
418+
},
419+
}
420+
conn = client._connection = _Connection(resource)
421+
table_ref = client.dataset(dataset_id).table(table_id)
422+
table = Table(table_ref, client=client)
423+
table.partitioning_type = 'DAY'
424+
425+
got = client.create_table(table)
426+
427+
self.assertEqual(len(conn._requested), 1)
428+
req = conn._requested[0]
429+
self.assertEqual(req['method'], 'POST')
430+
self.assertEqual(req['path'], '/%s' % path)
431+
sent = {
432+
'tableReference': {
433+
'projectId': project,
434+
'datasetId': dataset_id,
435+
'tableId': table_id
436+
},
437+
'timePartitioning': {'type': 'DAY'},
438+
}
439+
self.assertEqual(req['data'], sent)
440+
self.assertEqual(table.partitioning_type, "DAY")
441+
self.assertEqual(got.table_id, table_id)
442+
443+
def test_create_table_w_day_partition_and_expire(self):
444+
from google.cloud.bigquery.table import Table
445+
446+
project = 'PROJECT'
447+
dataset_id = 'dataset_id'
448+
table_id = 'table-id'
449+
path = 'projects/%s/datasets/%s/tables' % (
450+
project, dataset_id)
451+
creds = _make_credentials()
452+
client = self._make_one(project=project, credentials=creds)
453+
resource = {
454+
'id': '%s:%s:%s' % (project, dataset_id, table_id),
455+
'tableReference': {
456+
'projectId': project,
457+
'datasetId': dataset_id,
458+
'tableId': table_id
459+
},
460+
}
461+
conn = client._connection = _Connection(resource)
462+
table_ref = client.dataset(dataset_id).table(table_id)
463+
table = Table(table_ref, client=client)
464+
table.partitioning_type = 'DAY'
465+
table.partition_expiration = 100
466+
467+
got = client.create_table(table)
468+
469+
self.assertEqual(len(conn._requested), 1)
470+
req = conn._requested[0]
471+
self.assertEqual(req['method'], 'POST')
472+
self.assertEqual(req['path'], '/%s' % path)
473+
sent = {
474+
'tableReference': {
475+
'projectId': project,
476+
'datasetId': dataset_id,
477+
'tableId': table_id
478+
},
479+
'timePartitioning': {'type': 'DAY', 'expirationMs': 100},
480+
}
481+
self.assertEqual(req['data'], sent)
482+
self.assertEqual(table.partitioning_type, "DAY")
483+
self.assertEqual(table.partition_expiration, 100)
484+
self.assertEqual(got.table_id, table_id)
485+
486+
def test_create_table_w_schema_and_query(self):
487+
from google.cloud.bigquery.table import Table, SchemaField
488+
489+
project = 'PROJECT'
490+
dataset_id = 'dataset_id'
491+
table_id = 'table-id'
492+
path = 'projects/%s/datasets/%s/tables' % (
493+
project, dataset_id)
494+
query = 'SELECT * from %s:%s' % (dataset_id, table_id)
495+
creds = _make_credentials()
496+
client = self._make_one(project=project, credentials=creds)
497+
resource = {
498+
'id': '%s:%s:%s' % (project, dataset_id, table_id),
499+
'tableReference': {
500+
'projectId': project,
501+
'datasetId': dataset_id,
502+
'tableId': table_id
503+
},
504+
'schema': {'fields': [
505+
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
506+
{'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]
507+
},
508+
'view': {
509+
'query': query,
510+
'useLegacySql': True
511+
},
512+
}
513+
schema = [
514+
SchemaField('full_name', 'STRING', mode='REQUIRED'),
515+
SchemaField('age', 'INTEGER', mode='REQUIRED')
516+
]
517+
conn = client._connection = _Connection(resource)
518+
table_ref = client.dataset(dataset_id).table(table_id)
519+
table = Table(table_ref, schema=schema, client=client)
520+
table.view_query = query
521+
522+
got = client.create_table(table)
523+
524+
self.assertEqual(len(conn._requested), 1)
525+
req = conn._requested[0]
526+
self.assertEqual(req['method'], 'POST')
527+
self.assertEqual(req['path'], '/%s' % path)
528+
sent = {
529+
'tableReference': {
530+
'projectId': project,
531+
'datasetId': dataset_id,
532+
'tableId': table_id
533+
},
534+
'schema': {'fields': [
535+
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
536+
{'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]
537+
},
538+
'view': {'query': query},
539+
}
540+
self.assertEqual(req['data'], sent)
541+
self.assertEqual(got.table_id, table_id)
542+
self.assertEqual(got.project, project)
543+
self.assertEqual(got.dataset_id, dataset_id)
544+
self.assertEqual(got.schema, schema)
545+
self.assertEqual(got.view_query, query)
546+
402547
def test_get_table(self):
403548
project = 'PROJECT'
404549
dataset_id = 'dataset_id'

0 commit comments

Comments
 (0)