Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix handling of timestamps in BigQuery insert data method #1250

Merged
merged 2 commits into from
Dec 1, 2015
Merged

Fix handling of timestamps in BigQuery insert data method #1250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
46e8fd2
Updating BigQuery inserted timestamps to use seconds.
dhermes Nov 30, 2015
b592be7
Updating BigQuery system test to verify timestamp handling.
dhermes Nov 30, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions gcloud/bigquery/table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import six

from gcloud._helpers import _datetime_from_microseconds
from gcloud._helpers import _microseconds_from_datetime
from gcloud._helpers import _millis_from_datetime
from gcloud.exceptions import NotFound
from gcloud.bigquery._helpers import _rows_from_json
Expand Down Expand Up @@ -657,8 +658,11 @@ def insert_data(self,
row_info = {}

for field, value in zip(self._schema, row):
if field.field_type == 'TIMESTAMP':
value = _millis_from_datetime(value)
if field.field_type == 'TIMESTAMP' and value is not None:
# BigQuery stores TIMESTAMP data internally as a
# UNIX timestamp with microsecond precision.
# Specifies the number of seconds since the epoch.
value = _microseconds_from_datetime(value) * 1e-6
row_info[field.name] = value

info = {'json': row_info}
Expand Down
7 changes: 5 additions & 2 deletions gcloud/bigquery/test_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1060,7 +1060,7 @@ def test_fetch_data_w_record_schema(self):
def test_insert_data_w_bound_client(self):
import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _millis_from_datetime
from gcloud._helpers import _microseconds_from_datetime
from gcloud.bigquery.table import SchemaField

WHEN_TS = 1437767599.006
Expand All @@ -1084,9 +1084,12 @@ def test_insert_data_w_bound_client(self):
]

def _row_data(row):
joined = None
if row[2] is not None:
joined = _microseconds_from_datetime(row[2]) * 1e-6
return {'full_name': row[0],
'age': row[1],
'joined': _millis_from_datetime(row[2])}
'joined': joined}

SENT = {
'rows': [{'json': _row_data(row)} for row in ROWS],
Expand Down
17 changes: 12 additions & 5 deletions system_tests/bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,17 @@ def test_update_table(self):
self.assertEqual(found.mode, expected.mode)

def test_load_table_then_dump_table(self):
import datetime
from gcloud._helpers import UTC

NOW_SECONDS = 1448911495.484366
NOW = datetime.datetime.utcfromtimestamp(
NOW_SECONDS).replace(tzinfo=UTC)
ROWS = [
('Phred Phlyntstone', 32),
('Bharney Rhubble', 33),
('Wylma Phlyntstone', 29),
('Bhettye Rhubble', 27),
('Phred Phlyntstone', 32, NOW),
('Bharney Rhubble', 33, NOW + datetime.timedelta(seconds=10)),
('Wylma Phlyntstone', 29, NOW + datetime.timedelta(seconds=20)),
('Bhettye Rhubble', 27, None),
]
ROW_IDS = range(len(ROWS))
dataset = CLIENT.dataset(DATASET_NAME)
Expand All @@ -206,7 +212,8 @@ def test_load_table_then_dump_table(self):
full_name = bigquery.SchemaField('full_name', 'STRING',
mode='REQUIRED')
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
table = dataset.table(TABLE_NAME, schema=[full_name, age])
now = bigquery.SchemaField('now', 'TIMESTAMP')
table = dataset.table(TABLE_NAME, schema=[full_name, age, now])
self.assertFalse(table.exists())
table.create()
self.to_delete.insert(0, table)
Expand Down