From d4d5d4d80fd22dd6f648ac514daa6db71b23cfd8 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 16 Apr 2015 15:17:05 -0400 Subject: [PATCH 1/3] Move RFC 3339 timestamp format string up to 'gcloud._helpers'. Use it consistently in pubsub. --- gcloud/_helpers.py | 2 ++ gcloud/pubsub/message.py | 2 +- gcloud/pubsub/test_message.py | 2 +- gcloud/pubsub/test_topic.py | 13 +++---------- gcloud/pubsub/topic.py | 3 ++- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index ab730c947b2a..f62bb964c333 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -31,6 +31,8 @@ class Local(object): except ImportError: app_identity = None +_RFC3339_MICROS = '%Y-%m-%dT%H:%M:%S.%fZ' + class _LocalStack(Local): """Manage a thread-local LIFO stack of resources. diff --git a/gcloud/pubsub/message.py b/gcloud/pubsub/message.py index 1a04c3ee6fa7..b01c0e9c29b4 100644 --- a/gcloud/pubsub/message.py +++ b/gcloud/pubsub/message.py @@ -19,7 +19,7 @@ import pytz -_RFC3339_MICROS = '%Y-%m-%dT%H:%M:%S.%fZ' +from gcloud._helpers import _RFC3339_MICROS class Message(object): diff --git a/gcloud/pubsub/test_message.py b/gcloud/pubsub/test_message.py index 6cf97f677ec2..38ad240e6199 100644 --- a/gcloud/pubsub/test_message.py +++ b/gcloud/pubsub/test_message.py @@ -92,7 +92,7 @@ def _to_fail(): def test_timestamp_w_timestamp_in_attributes(self): from datetime import datetime from pytz import utc - from gcloud.pubsub.message import _RFC3339_MICROS + from gcloud._helpers import _RFC3339_MICROS DATA = b'DEADBEEF' MESSAGE_ID = b'12345' TIMESTAMP = '2015-04-10T18:42:27.131956Z' diff --git a/gcloud/pubsub/test_topic.py b/gcloud/pubsub/test_topic.py index d66cf6d68c05..932eb2f5229f 100644 --- a/gcloud/pubsub/test_topic.py +++ b/gcloud/pubsub/test_topic.py @@ -155,6 +155,7 @@ def test_publish_single_bytes_wo_attrs_w_add_timestamp(self): import base64 import datetime from gcloud.pubsub import topic as MUT + from gcloud._helpers import _RFC3339_MICROS from gcloud._testing import _Monkey NOW = datetime.datetime.utcnow() @@ -167,7 +168,7 @@ def _utcnow(): B64 = base64.b64encode(PAYLOAD).decode('ascii') MSGID = 'DEADBEEF' MESSAGE = {'data': B64, - 'attributes': {'timestamp': '%sZ' % NOW.isoformat()}} + 'attributes': {'timestamp': NOW.strftime(_RFC3339_MICROS)}} PATH = 'projects/%s/topics/%s' % (PROJECT, TOPIC_NAME) conn = _Connection({'messageIds': [MSGID]}) topic = self._makeOne(TOPIC_NAME, project=PROJECT, connection=conn, @@ -183,13 +184,6 @@ def _utcnow(): def test_publish_single_bytes_w_add_timestamp_w_ts_in_attrs(self): import base64 - import datetime - from gcloud.pubsub import topic as MUT - from gcloud._testing import _Monkey - NOW = datetime.datetime.utcnow() - - def _utcnow(): # pragma: NO COVER - return NOW TOPIC_NAME = 'topic_name' PROJECT = 'PROJECT' @@ -203,8 +197,7 @@ def _utcnow(): # pragma: NO COVER conn = _Connection({'messageIds': [MSGID]}) topic = self._makeOne(TOPIC_NAME, project=PROJECT, connection=conn, timestamp_messages=True) - with _Monkey(MUT, _NOW=_utcnow): - msgid = topic.publish(PAYLOAD, timestamp=OVERRIDE) + msgid = topic.publish(PAYLOAD, timestamp=OVERRIDE) self.assertEqual(msgid, MSGID) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] diff --git a/gcloud/pubsub/topic.py b/gcloud/pubsub/topic.py index 67814fdb7ec2..98a3a74a4550 100644 --- a/gcloud/pubsub/topic.py +++ b/gcloud/pubsub/topic.py @@ -18,6 +18,7 @@ import datetime from gcloud._helpers import get_default_project +from gcloud._helpers import _RFC3339_MICROS from gcloud.exceptions import NotFound from gcloud.pubsub._implicit_environ import get_default_connection @@ -123,7 +124,7 @@ def publish(self, message, **attrs): :returns: message ID assigned by the server to the published message """ if self.timestamp_messages and 'timestamp' not in attrs: - attrs['timestamp'] = '%sZ' % _NOW().isoformat() + attrs['timestamp'] = _NOW().strftime(_RFC3339_MICROS) message_b = base64.b64encode(message).decode('ascii') message_data = {'data': message_b, 'attributes': attrs} data = {'messages': [message_data]} From 5d13cbe46897b5f258db7ea11947f78b0e26282a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 16 Apr 2015 15:23:35 -0400 Subject: [PATCH 2/3] Use shared RFC 3339 timestamp format string in storage. --- gcloud/storage/blob.py | 6 +++--- gcloud/storage/bucket.py | 4 ++-- gcloud/storage/test_blob.py | 6 ++++-- gcloud/storage/test_bucket.py | 3 ++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 74e934813ec7..4eadff89aa2f 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -35,10 +35,10 @@ from gcloud.storage._helpers import _scalar_property from gcloud.storage import _implicit_environ from gcloud.storage.acl import ObjectACL +from gcloud._helpers import _RFC3339_MICROS _API_ACCESS_ENDPOINT = 'https://storage.googleapis.com' -_GOOGLE_TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' class Blob(_PropertyMixin): @@ -749,7 +749,7 @@ def time_deleted(self): """ value = self._properties.get('timeDeleted') if value is not None: - return datetime.datetime.strptime(value, _GOOGLE_TIMESTAMP_FORMAT) + return datetime.datetime.strptime(value, _RFC3339_MICROS) @property def updated(self): @@ -763,7 +763,7 @@ def updated(self): """ value = self._properties.get('updated') if value is not None: - return datetime.datetime.strptime(value, _GOOGLE_TIMESTAMP_FORMAT) + return datetime.datetime.strptime(value, _RFC3339_MICROS) class _UploadConfig(object): diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 023c7440a3c4..d4ae8bb5a044 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -46,7 +46,7 @@ from gcloud.storage.acl import DefaultObjectACL from gcloud.storage.iterator import Iterator from gcloud.storage.blob import Blob -from gcloud.storage.blob import _GOOGLE_TIMESTAMP_FORMAT +from gcloud._helpers import _RFC3339_MICROS class _BlobIterator(Iterator): @@ -693,7 +693,7 @@ def time_created(self): """ value = self._properties.get('timeCreated') if value is not None: - return datetime.datetime.strptime(value, _GOOGLE_TIMESTAMP_FORMAT) + return datetime.datetime.strptime(value, _RFC3339_MICROS) @property def versioning_enabled(self): diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index 3df74e7efe4e..08bbbdc77117 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -1016,11 +1016,12 @@ def test_storage_class(self): def test_time_deleted(self): import datetime + from gcloud._helpers import _RFC3339_MICROS BLOB_NAME = 'blob-name' connection = _Connection() bucket = _Bucket(connection) TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37) - TIME_DELETED = TIMESTAMP.isoformat() + '.000Z' + TIME_DELETED = TIMESTAMP.strftime(_RFC3339_MICROS) properties = {'timeDeleted': TIME_DELETED} blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties) self.assertEqual(blob.time_deleted, TIMESTAMP) @@ -1032,11 +1033,12 @@ def test_time_deleted_unset(self): def test_updated(self): import datetime + from gcloud._helpers import _RFC3339_MICROS BLOB_NAME = 'blob-name' connection = _Connection() bucket = _Bucket(connection) TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37) - UPDATED = TIMESTAMP.isoformat() + '.000Z' + UPDATED = TIMESTAMP.strftime(_RFC3339_MICROS) properties = {'updated': UPDATED} blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties) self.assertEqual(blob.updated, TIMESTAMP) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index c56c12745d0e..43273aafc955 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -790,8 +790,9 @@ def test_storage_class(self): def test_time_created(self): import datetime + from gcloud._helpers import _RFC3339_MICROS TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37) - TIME_CREATED = TIMESTAMP.isoformat() + '.000Z' + TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) properties = {'timeCreated': TIME_CREATED} bucket = self._makeOne(properties=properties) self.assertEqual(bucket.time_created, TIMESTAMP) From 1d35fe86b34317be3865712a20339ea8524452d9 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 16 Apr 2015 17:34:04 -0400 Subject: [PATCH 3/3] Don't return naive datetimes from bucket/blob timestamp accessors. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/835#discussion_r28548332 --- gcloud/storage/blob.py | 7 +++++-- gcloud/storage/bucket.py | 5 ++++- gcloud/storage/test_blob.py | 6 ++++-- gcloud/storage/test_bucket.py | 3 ++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 4eadff89aa2f..2926ec2992f6 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -22,6 +22,7 @@ import os import time +import pytz import six from six.moves.urllib.parse import quote # pylint: disable=F0401 @@ -749,7 +750,8 @@ def time_deleted(self): """ value = self._properties.get('timeDeleted') if value is not None: - return datetime.datetime.strptime(value, _RFC3339_MICROS) + naive = datetime.datetime.strptime(value, _RFC3339_MICROS) + return naive.replace(tzinfo=pytz.utc) @property def updated(self): @@ -763,7 +765,8 @@ def updated(self): """ value = self._properties.get('updated') if value is not None: - return datetime.datetime.strptime(value, _RFC3339_MICROS) + naive = datetime.datetime.strptime(value, _RFC3339_MICROS) + return naive.replace(tzinfo=pytz.utc) class _UploadConfig(object): diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index d4ae8bb5a044..d45383b19ffc 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -36,6 +36,8 @@ import datetime import copy import os + +import pytz import six from gcloud._helpers import get_default_project @@ -693,7 +695,8 @@ def time_created(self): """ value = self._properties.get('timeCreated') if value is not None: - return datetime.datetime.strptime(value, _RFC3339_MICROS) + naive = datetime.datetime.strptime(value, _RFC3339_MICROS) + return naive.replace(tzinfo=pytz.utc) @property def versioning_enabled(self): diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index 08bbbdc77117..07791862ba53 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -1016,11 +1016,12 @@ def test_storage_class(self): def test_time_deleted(self): import datetime + from pytz import utc from gcloud._helpers import _RFC3339_MICROS BLOB_NAME = 'blob-name' connection = _Connection() bucket = _Bucket(connection) - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37) + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=utc) TIME_DELETED = TIMESTAMP.strftime(_RFC3339_MICROS) properties = {'timeDeleted': TIME_DELETED} blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties) @@ -1033,11 +1034,12 @@ def test_time_deleted_unset(self): def test_updated(self): import datetime + from pytz import utc from gcloud._helpers import _RFC3339_MICROS BLOB_NAME = 'blob-name' connection = _Connection() bucket = _Bucket(connection) - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37) + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=utc) UPDATED = TIMESTAMP.strftime(_RFC3339_MICROS) properties = {'updated': UPDATED} blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 43273aafc955..c9ae0e56b320 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -790,8 +790,9 @@ def test_storage_class(self): def test_time_created(self): import datetime + from pytz import utc from gcloud._helpers import _RFC3339_MICROS - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37) + TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=utc) TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) properties = {'timeCreated': TIME_CREATED} bucket = self._makeOne(properties=properties)