From c3c9a7a04dede100edaaf5b23ff6e855e84f75e0 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Sun, 31 Jan 2016 12:31:08 -0800 Subject: [PATCH 01/11] added support for generation-based query and delete of blobs. --- gcloud/storage/bucket.py | 30 ++++++++++++++++--- gcloud/storage/test_bucket.py | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 589f8527fb3f..e8cc43ef4455 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -17,6 +17,7 @@ import copy import six +from six.moves.urllib.parse import quote # pylint: disable=F0401 from gcloud._helpers import _rfc3339_to_datetime from gcloud.exceptions import NotFound @@ -202,7 +203,7 @@ def path(self): return self.path_helper(self.name) - def get_blob(self, blob_name, client=None): + def get_blob(self, blob_name, client=None, generation=None): """Get a blob object by name. This will return None if the blob doesn't exist:: @@ -214,6 +215,8 @@ def get_blob(self, blob_name, client=None): >>> print bucket.get_blob('/does-not-exist.txt') None + >>> print bucket.get_blob('/path/to/versioned_blob.txt', generation=generation_id) + :type blob_name: string :param blob_name: The name of the blob to retrieve. @@ -222,14 +225,21 @@ def get_blob(self, blob_name, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type generation: int + :param generation: Optional. The generation id to retrieve in a bucket + that supports versioning. + :rtype: :class:`gcloud.storage.blob.Blob` or None :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) blob = Blob(bucket=self, name=blob_name) try: + path = blob.path + if generation is not None: + path = blob.path + '?generation=' + quote(str(generation), safe='') response = client.connection.api_request( - method='GET', path=blob.path, _target_object=blob) + method='GET', path=path, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when @@ -357,7 +367,7 @@ def delete(self, force=False, client=None): client.connection.api_request(method='DELETE', path=self.path, _target_object=None) - def delete_blob(self, blob_name, client=None): + def delete_blob(self, blob_name, client=None, generation=None): """Deletes a blob from the current bucket. If the blob isn't found (backend 404), raises a @@ -384,6 +394,10 @@ def delete_blob(self, blob_name, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type generation: int + :param generation: Optional. The generation of this object to delete. + Only works on buckets with versioning enabled. + :raises: :class:`gcloud.exceptions.NotFound` (to suppress the exception, call ``delete_blobs``, passing a no-op ``on_error`` callback, e.g.:: @@ -392,6 +406,9 @@ def delete_blob(self, blob_name, client=None): """ client = self._require_client(client) blob_path = Blob.path_helper(self.path, blob_name) + if generation is not None: + blob_path = blob_path + '?generation=' + quote(str(generation), safe='') + # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or # in a batch request). @@ -423,7 +440,12 @@ def delete_blobs(self, blobs, on_error=None, client=None): blob_name = blob if not isinstance(blob_name, six.string_types): blob_name = blob.name - self.delete_blob(blob_name, client=client) + + generation = None + if hasattr(blob,'generation'): + generation = blob.generation + + self.delete_blob(blob_name, client=client, generation=generation) except NotFound: if on_error is not None: on_error(blob) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 947b1f0e600d..ad6e2b9636bc 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -238,6 +238,34 @@ def test_get_blob_hit(self): self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + def test_get_blob_miss_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-w-version' + GENERATION = 999 + connection = _Connection({'name': BLOB_NAME, 'generation': 1}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob, None) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + + def test_get_blob_hit_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-w-version' + GENERATION = 999 + connection = _Connection({'name': BLOB_NAME, 'generation': GENERATION}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob.bucket is bucket) + self.assertEqual(blob.name, BLOB_NAME) + self.assertEqual(blob.generation, GENERATION) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%s' % (NAME, BLOB_NAME, GENERATION)) + def test_list_blobs_defaults(self): NAME = 'name' connection = _Connection({'items': []}) @@ -419,6 +447,32 @@ def test_delete_blob_hit(self): self.assertEqual(kw['method'], 'DELETE') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + def test_delete_blob_miss_w_generation(self): + from gcloud.exceptions import NotFound + NAME = 'name' + BLOB_NAME = 'blob_name' + GENERATION = 1 + connection = _Connection() + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + self.assertRaises(NotFound, bucket.delete_blob, BLOB_NAME, generation=GENERATION) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + + def test_delete_blob_hit_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name' + GENERATION = 999 + connection = _Connection({}) + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) + self.assertTrue(result is None) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + def test_delete_blobs_empty(self): NAME = 'name' connection = _Connection() From 35cf68867a85e719765cb6f098b21d6c9ee17ece Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Sun, 31 Jan 2016 12:31:08 -0800 Subject: [PATCH 02/11] added support for generation-based query and delete of blobs. --- gcloud/storage/bucket.py | 30 ++++++++++++++++--- gcloud/storage/test_bucket.py | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 589f8527fb3f..e8cc43ef4455 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -17,6 +17,7 @@ import copy import six +from six.moves.urllib.parse import quote # pylint: disable=F0401 from gcloud._helpers import _rfc3339_to_datetime from gcloud.exceptions import NotFound @@ -202,7 +203,7 @@ def path(self): return self.path_helper(self.name) - def get_blob(self, blob_name, client=None): + def get_blob(self, blob_name, client=None, generation=None): """Get a blob object by name. This will return None if the blob doesn't exist:: @@ -214,6 +215,8 @@ def get_blob(self, blob_name, client=None): >>> print bucket.get_blob('/does-not-exist.txt') None + >>> print bucket.get_blob('/path/to/versioned_blob.txt', generation=generation_id) + :type blob_name: string :param blob_name: The name of the blob to retrieve. @@ -222,14 +225,21 @@ def get_blob(self, blob_name, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type generation: int + :param generation: Optional. The generation id to retrieve in a bucket + that supports versioning. + :rtype: :class:`gcloud.storage.blob.Blob` or None :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) blob = Blob(bucket=self, name=blob_name) try: + path = blob.path + if generation is not None: + path = blob.path + '?generation=' + quote(str(generation), safe='') response = client.connection.api_request( - method='GET', path=blob.path, _target_object=blob) + method='GET', path=path, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when @@ -357,7 +367,7 @@ def delete(self, force=False, client=None): client.connection.api_request(method='DELETE', path=self.path, _target_object=None) - def delete_blob(self, blob_name, client=None): + def delete_blob(self, blob_name, client=None, generation=None): """Deletes a blob from the current bucket. If the blob isn't found (backend 404), raises a @@ -384,6 +394,10 @@ def delete_blob(self, blob_name, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type generation: int + :param generation: Optional. The generation of this object to delete. + Only works on buckets with versioning enabled. + :raises: :class:`gcloud.exceptions.NotFound` (to suppress the exception, call ``delete_blobs``, passing a no-op ``on_error`` callback, e.g.:: @@ -392,6 +406,9 @@ def delete_blob(self, blob_name, client=None): """ client = self._require_client(client) blob_path = Blob.path_helper(self.path, blob_name) + if generation is not None: + blob_path = blob_path + '?generation=' + quote(str(generation), safe='') + # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or # in a batch request). @@ -423,7 +440,12 @@ def delete_blobs(self, blobs, on_error=None, client=None): blob_name = blob if not isinstance(blob_name, six.string_types): blob_name = blob.name - self.delete_blob(blob_name, client=client) + + generation = None + if hasattr(blob,'generation'): + generation = blob.generation + + self.delete_blob(blob_name, client=client, generation=generation) except NotFound: if on_error is not None: on_error(blob) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 947b1f0e600d..ad6e2b9636bc 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -238,6 +238,34 @@ def test_get_blob_hit(self): self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + def test_get_blob_miss_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-w-version' + GENERATION = 999 + connection = _Connection({'name': BLOB_NAME, 'generation': 1}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob, None) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + + def test_get_blob_hit_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-w-version' + GENERATION = 999 + connection = _Connection({'name': BLOB_NAME, 'generation': GENERATION}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob.bucket is bucket) + self.assertEqual(blob.name, BLOB_NAME) + self.assertEqual(blob.generation, GENERATION) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%s' % (NAME, BLOB_NAME, GENERATION)) + def test_list_blobs_defaults(self): NAME = 'name' connection = _Connection({'items': []}) @@ -419,6 +447,32 @@ def test_delete_blob_hit(self): self.assertEqual(kw['method'], 'DELETE') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + def test_delete_blob_miss_w_generation(self): + from gcloud.exceptions import NotFound + NAME = 'name' + BLOB_NAME = 'blob_name' + GENERATION = 1 + connection = _Connection() + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + self.assertRaises(NotFound, bucket.delete_blob, BLOB_NAME, generation=GENERATION) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + + def test_delete_blob_hit_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name' + GENERATION = 999 + connection = _Connection({}) + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) + self.assertTrue(result is None) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + def test_delete_blobs_empty(self): NAME = 'name' connection = _Connection() From 016080aae10f6acb9d9073e3a5abf995eac892cc Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Sun, 31 Jan 2016 21:14:02 -0800 Subject: [PATCH 03/11] cleaned up to past lint. --- gcloud/storage/bucket.py | 15 ++++++++++----- gcloud/storage/test_bucket.py | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index e8cc43ef4455..46e2942f0ac6 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -215,7 +215,9 @@ def get_blob(self, blob_name, client=None, generation=None): >>> print bucket.get_blob('/does-not-exist.txt') None - >>> print bucket.get_blob('/path/to/versioned_blob.txt', generation=generation_id) + >>> print bucket.get_blob( + ... '/path/to/versioned_blob.txt', + ... generation=generation_id) :type blob_name: string @@ -237,7 +239,8 @@ def get_blob(self, blob_name, client=None, generation=None): try: path = blob.path if generation is not None: - path = blob.path + '?generation=' + quote(str(generation), safe='') + path = (blob.path + '?generation=' + + quote(str(generation), safe='')) response = client.connection.api_request( method='GET', path=path, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. @@ -407,7 +410,8 @@ def delete_blob(self, blob_name, client=None, generation=None): client = self._require_client(client) blob_path = Blob.path_helper(self.path, blob_name) if generation is not None: - blob_path = blob_path + '?generation=' + quote(str(generation), safe='') + blob_path = (blob_path + '?generation=' + + quote(str(generation), safe='')) # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or @@ -442,10 +446,11 @@ def delete_blobs(self, blobs, on_error=None, client=None): blob_name = blob.name generation = None - if hasattr(blob,'generation'): + if hasattr(blob, 'generation'): generation = blob.generation - self.delete_blob(blob_name, client=client, generation=generation) + self.delete_blob(blob_name, client=client, + generation=generation) except NotFound: if on_error is not None: on_error(blob) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index ad6e2b9636bc..44482769a2b6 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -249,7 +249,8 @@ def test_get_blob_miss_w_generation(self): self.assertTrue(blob, None) kw, = connection._requested self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % + (NAME, BLOB_NAME, GENERATION)) def test_get_blob_hit_w_generation(self): NAME = 'name' @@ -264,7 +265,8 @@ def test_get_blob_hit_w_generation(self): self.assertEqual(blob.generation, GENERATION) kw, = connection._requested self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%s' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%s' % + (NAME, BLOB_NAME, GENERATION)) def test_list_blobs_defaults(self): NAME = 'name' @@ -455,10 +457,12 @@ def test_delete_blob_miss_w_generation(self): connection = _Connection() client = _Client(connection) bucket = self._makeOne(client=client, name=NAME) - self.assertRaises(NotFound, bucket.delete_blob, BLOB_NAME, generation=GENERATION) + self.assertRaises(NotFound, bucket.delete_blob, + BLOB_NAME, generation=GENERATION) kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % + (NAME, BLOB_NAME, GENERATION)) def test_delete_blob_hit_w_generation(self): NAME = 'name' @@ -471,7 +475,8 @@ def test_delete_blob_hit_w_generation(self): self.assertTrue(result is None) kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % + (NAME, BLOB_NAME, GENERATION)) def test_delete_blobs_empty(self): NAME = 'name' From 299fab9387f450c0155086c768b4a28fc22d9792 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Sun, 31 Jan 2016 21:14:02 -0800 Subject: [PATCH 04/11] cleaned up to pass lint. --- gcloud/storage/bucket.py | 15 ++++++++++----- gcloud/storage/test_bucket.py | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index e8cc43ef4455..46e2942f0ac6 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -215,7 +215,9 @@ def get_blob(self, blob_name, client=None, generation=None): >>> print bucket.get_blob('/does-not-exist.txt') None - >>> print bucket.get_blob('/path/to/versioned_blob.txt', generation=generation_id) + >>> print bucket.get_blob( + ... '/path/to/versioned_blob.txt', + ... generation=generation_id) :type blob_name: string @@ -237,7 +239,8 @@ def get_blob(self, blob_name, client=None, generation=None): try: path = blob.path if generation is not None: - path = blob.path + '?generation=' + quote(str(generation), safe='') + path = (blob.path + '?generation=' + + quote(str(generation), safe='')) response = client.connection.api_request( method='GET', path=path, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. @@ -407,7 +410,8 @@ def delete_blob(self, blob_name, client=None, generation=None): client = self._require_client(client) blob_path = Blob.path_helper(self.path, blob_name) if generation is not None: - blob_path = blob_path + '?generation=' + quote(str(generation), safe='') + blob_path = (blob_path + '?generation=' + + quote(str(generation), safe='')) # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or @@ -442,10 +446,11 @@ def delete_blobs(self, blobs, on_error=None, client=None): blob_name = blob.name generation = None - if hasattr(blob,'generation'): + if hasattr(blob, 'generation'): generation = blob.generation - self.delete_blob(blob_name, client=client, generation=generation) + self.delete_blob(blob_name, client=client, + generation=generation) except NotFound: if on_error is not None: on_error(blob) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index ad6e2b9636bc..44482769a2b6 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -249,7 +249,8 @@ def test_get_blob_miss_w_generation(self): self.assertTrue(blob, None) kw, = connection._requested self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % + (NAME, BLOB_NAME, GENERATION)) def test_get_blob_hit_w_generation(self): NAME = 'name' @@ -264,7 +265,8 @@ def test_get_blob_hit_w_generation(self): self.assertEqual(blob.generation, GENERATION) kw, = connection._requested self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%s' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%s' % + (NAME, BLOB_NAME, GENERATION)) def test_list_blobs_defaults(self): NAME = 'name' @@ -455,10 +457,12 @@ def test_delete_blob_miss_w_generation(self): connection = _Connection() client = _Client(connection) bucket = self._makeOne(client=client, name=NAME) - self.assertRaises(NotFound, bucket.delete_blob, BLOB_NAME, generation=GENERATION) + self.assertRaises(NotFound, bucket.delete_blob, + BLOB_NAME, generation=GENERATION) kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % + (NAME, BLOB_NAME, GENERATION)) def test_delete_blob_hit_w_generation(self): NAME = 'name' @@ -471,7 +475,8 @@ def test_delete_blob_hit_w_generation(self): self.assertTrue(result is None) kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % + (NAME, BLOB_NAME, GENERATION)) def test_delete_blobs_empty(self): NAME = 'name' From e1e02dac1f06e99a1dd7431b258c1c7bac07ced0 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Mon, 1 Feb 2016 16:14:10 -0800 Subject: [PATCH 05/11] using 'client.connection.api_request' instead of manually creating API paths. added generation support to blob.delete and blob.exists. --- gcloud/storage/blob.py | 6 ++++-- gcloud/storage/bucket.py | 16 ++++++---------- gcloud/storage/test_blob.py | 34 ++++++++++++++++++++++++++++++++++ gcloud/storage/test_bucket.py | 12 ++++-------- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 2947cef50761..43f6356eddc3 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -240,7 +240,8 @@ def exists(self, client=None): try: # We only need the status code (200 or not) so we seek to # minimize the returned payload. - query_params = {'fields': 'name'} + query_params = {'fields': 'name', 'generation': self.generation} + # We intentionally pass `_target_object=None` since fields=name # would limit the local properties. client.connection.api_request(method='GET', path=self.path, @@ -266,7 +267,8 @@ def delete(self, client=None): (propagated from :meth:`gcloud.storage.bucket.Bucket.delete_blob`). """ - return self.bucket.delete_blob(self.name, client=client) + return self.bucket.delete_blob(self.name, client=client, + generation=self.generation) def download_to_file(self, file_obj, client=None): """Download the contents of this blob into a file-like object. diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 46e2942f0ac6..b9e8a2dff57c 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -17,7 +17,6 @@ import copy import six -from six.moves.urllib.parse import quote # pylint: disable=F0401 from gcloud._helpers import _rfc3339_to_datetime from gcloud.exceptions import NotFound @@ -237,12 +236,10 @@ def get_blob(self, blob_name, client=None, generation=None): client = self._require_client(client) blob = Blob(bucket=self, name=blob_name) try: - path = blob.path - if generation is not None: - path = (blob.path + '?generation=' + - quote(str(generation), safe='')) + query_params = {'generation': generation} response = client.connection.api_request( - method='GET', path=path, _target_object=blob) + method='GET', path=blob.path, + query_params=query_params, target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when @@ -409,15 +406,14 @@ def delete_blob(self, blob_name, client=None, generation=None): """ client = self._require_client(client) blob_path = Blob.path_helper(self.path, blob_name) - if generation is not None: - blob_path = (blob_path + '?generation=' + - quote(str(generation), safe='')) + query_params = {'generation': generation} # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or # in a batch request). client.connection.api_request(method='DELETE', path=blob_path, - _target_object=None) + query_params=query_params, + target_object=None) def delete_blobs(self, blobs, on_error=None, client=None): """Deletes a list of blobs from the current bucket. diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index 2b958e6d4308..bcb76766b892 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -241,6 +241,22 @@ def test_exists_hit(self): bucket._blobs[BLOB_NAME] = 1 self.assertTrue(blob.exists()) + def test_exists_hit_w_generation(self): + from six.moves.http_client import OK + BLOB_NAME = 'blob-name' + GENERATION = 999 + found_response = {'status': OK} + connection = _Connection(found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + self.assertTrue(blob.exists()) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['query_params']['generation'], GENERATION) + def test_delete(self): from six.moves.http_client import NOT_FOUND BLOB_NAME = 'blob-name' @@ -254,6 +270,24 @@ def test_delete(self): self.assertFalse(blob.exists()) self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + def test_delete_w_generation(self): + from six.moves.http_client import NOT_FOUND + BLOB_NAME = 'blob-name' + GENERATION = 999 + not_found_response = {'status': NOT_FOUND} + connection = _Connection(not_found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + blob.delete() + self.assertFalse(blob.exists()) + self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['query_params']['generation'], GENERATION) + def _download_to_file_helper(self, chunk_size=None): from six.moves.http_client import OK from six.moves.http_client import PARTIAL_CONTENT diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 44482769a2b6..e0794ca31264 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -249,8 +249,7 @@ def test_get_blob_miss_w_generation(self): self.assertTrue(blob, None) kw, = connection._requested self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % - (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['query_params'], {'generation': GENERATION}) def test_get_blob_hit_w_generation(self): NAME = 'name' @@ -265,8 +264,7 @@ def test_get_blob_hit_w_generation(self): self.assertEqual(blob.generation, GENERATION) kw, = connection._requested self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%s' % - (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['query_params'], {'generation': GENERATION}) def test_list_blobs_defaults(self): NAME = 'name' @@ -461,8 +459,7 @@ def test_delete_blob_miss_w_generation(self): BLOB_NAME, generation=GENERATION) kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % - (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['query_params'], {'generation': GENERATION}) def test_delete_blob_hit_w_generation(self): NAME = 'name' @@ -475,8 +472,7 @@ def test_delete_blob_hit_w_generation(self): self.assertTrue(result is None) kw, = connection._requested self.assertEqual(kw['method'], 'DELETE') - self.assertEqual(kw['path'], '/b/%s/o/%s?generation=%d' % - (NAME, BLOB_NAME, GENERATION)) + self.assertEqual(kw['query_params'], {'generation': GENERATION}) def test_delete_blobs_empty(self): NAME = 'name' From 3aa118511ae0bb3487cd67ca1aeb6ae900f9cd72 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Tue, 2 Feb 2016 08:27:03 -0800 Subject: [PATCH 06/11] ignoring placeholder generation parameter in the mock delete_blob function --- gcloud/storage/test_blob.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index bcb76766b892..51d3900c7d68 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -1129,7 +1129,8 @@ def __init__(self, client=None): self._copied = [] self._deleted = [] - def delete_blob(self, blob_name, client=None): + def delete_blob(self, blob_name, client=None, + generation=None): # pylint: disable=W0613 del self._blobs[blob_name] self._deleted.append((blob_name, client)) From 70a4f4c6b347e26d55f422c09327d2241fe7dd40 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Wed, 3 Feb 2016 11:57:48 -0800 Subject: [PATCH 07/11] target_object parameter fix. --- gcloud/storage/bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index b9e8a2dff57c..c0973c8a0f39 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -239,7 +239,7 @@ def get_blob(self, blob_name, client=None, generation=None): query_params = {'generation': generation} response = client.connection.api_request( method='GET', path=blob.path, - query_params=query_params, target_object=blob) + query_params=query_params, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when @@ -413,7 +413,7 @@ def delete_blob(self, blob_name, client=None, generation=None): # in a batch request). client.connection.api_request(method='DELETE', path=blob_path, query_params=query_params, - target_object=None) + _target_object=None) def delete_blobs(self, blobs, on_error=None, client=None): """Deletes a list of blobs from the current bucket. From 9cc73fd2fe7e3ee1920dbd7e3537c28065bd3eff Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Wed, 3 Feb 2016 12:05:57 -0800 Subject: [PATCH 08/11] query_params are default empty if generation is None. --- gcloud/storage/blob.py | 4 +++- gcloud/storage/bucket.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 43f6356eddc3..394702136520 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -240,7 +240,9 @@ def exists(self, client=None): try: # We only need the status code (200 or not) so we seek to # minimize the returned payload. - query_params = {'fields': 'name', 'generation': self.generation} + query_params = {'fields': 'name'} + if self.generation is not None: + query_params['generation'] = self.generation # We intentionally pass `_target_object=None` since fields=name # would limit the local properties. diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index c0973c8a0f39..36f6a1be4427 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -236,7 +236,9 @@ def get_blob(self, blob_name, client=None, generation=None): client = self._require_client(client) blob = Blob(bucket=self, name=blob_name) try: - query_params = {'generation': generation} + query_params = {} + if generation is not None: + query_params = {'generation': generation} response = client.connection.api_request( method='GET', path=blob.path, query_params=query_params, _target_object=blob) @@ -406,7 +408,9 @@ def delete_blob(self, blob_name, client=None, generation=None): """ client = self._require_client(client) blob_path = Blob.path_helper(self.path, blob_name) - query_params = {'generation': generation} + query_params = {} + if generation is not None: + query_params = {'generation': generation} # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or From ed94125c94008c2eefa56303f847d973accec752 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Wed, 3 Feb 2016 12:37:40 -0800 Subject: [PATCH 09/11] added tests for None generations --- gcloud/storage/test_blob.py | 34 ++++++++++++++++++++++++++++++++++ gcloud/storage/test_bucket.py | 27 +++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index 51d3900c7d68..652e6f385691 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -257,6 +257,22 @@ def test_exists_hit_w_generation(self): self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['query_params']['generation'], GENERATION) + def test_exists_hit_w_none_generation(self): + from six.moves.http_client import OK + BLOB_NAME = 'blob-name' + GENERATION = None + found_response = {'status': OK} + connection = _Connection(found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + self.assertTrue(blob.exists()) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertNotIn('generation', kw['query_params']) + def test_delete(self): from six.moves.http_client import NOT_FOUND BLOB_NAME = 'blob-name' @@ -288,6 +304,24 @@ def test_delete_w_generation(self): self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['query_params']['generation'], GENERATION) + def test_delete_w_none_generation(self): + from six.moves.http_client import NOT_FOUND + BLOB_NAME = 'blob-name' + GENERATION = None + not_found_response = {'status': NOT_FOUND} + connection = _Connection(not_found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + blob.delete() + self.assertFalse(blob.exists()) + self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertNotIn('generation', kw['query_params']) + def _download_to_file_helper(self, chunk_size=None): from six.moves.http_client import OK from six.moves.http_client import PARTIAL_CONTENT diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index e0794ca31264..4c9c27c8545f 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -266,6 +266,20 @@ def test_get_blob_hit_w_generation(self): self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['query_params'], {'generation': GENERATION}) + def test_get_blob_hit_w_none_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-wo-version' + GENERATION = None + connection = _Connection({'name': BLOB_NAME}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob.bucket is bucket) + self.assertEqual(blob.name, BLOB_NAME) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertNotIn('generation', kw['query_params']) + def test_list_blobs_defaults(self): NAME = 'name' connection = _Connection({'items': []}) @@ -474,6 +488,19 @@ def test_delete_blob_hit_w_generation(self): self.assertEqual(kw['method'], 'DELETE') self.assertEqual(kw['query_params'], {'generation': GENERATION}) + def test_delete_blob_hit_w_none_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name' + GENERATION = None + connection = _Connection({}) + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) + self.assertTrue(result is None) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertNotIn('generation', kw['query_params']) + def test_delete_blobs_empty(self): NAME = 'name' connection = _Connection() From 8d92cf55264ad8d2295103c1b320ee8cbe56e7c1 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Wed, 3 Feb 2016 13:33:56 -0800 Subject: [PATCH 10/11] working on support for version sensitive renaming of blobs. --- gcloud/storage/bucket.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 36f6a1be4427..277fa2eacbf6 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -458,7 +458,7 @@ def delete_blobs(self, blobs, on_error=None, client=None): raise def copy_blob(self, blob, destination_bucket, new_name=None, - client=None): + client=None, versions=False): """Copy the given blob to the given bucket, optionally with a new name. :type blob: :class:`gcloud.storage.blob.Blob` @@ -475,16 +475,32 @@ def copy_blob(self, blob, destination_bucket, new_name=None, :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type versions: boolean + :param versions: Optional. Copy each version. + :rtype: :class:`gcloud.storage.blob.Blob` :returns: The new Blob. """ client = self._require_client(client) if new_name is None: new_name = blob.name + new_blob = Blob(bucket=destination_bucket, name=new_name) api_path = blob.path + '/copyTo' + new_blob.path - copy_result = client.connection.api_request( - method='POST', path=api_path, _target_object=new_blob) + query_params = {} + + # TODO(tsinha): Support multi-page results from list_blobs + if versions == True: + blob_versions = list(self.list_blobs(prefix=blob.name, versions=True, fields='name,generation,updated')) + for blob_version in blob_versions: + copy_result = client.connection.api_request( + method='POST', path=api_path, + query_params={'sourceGeneration': blob_version.generation}, + _target_object=new_blob) + else: + copy_result = client.connection.api_request( + method='POST', path=api_path, _target_object=new_blob) + new_blob._set_properties(copy_result) return new_blob @@ -513,7 +529,7 @@ def rename_blob(self, blob, new_name, client=None): :rtype: :class:`Blob` :returns: The newly-renamed blob. """ - new_blob = self.copy_blob(blob, self, new_name, client=client) + new_blob = self.copy_blob(blob, self, new_name, client=client, versions=True) blob.delete(client=client) return new_blob From 3bfdd10e586ecd765b83c2e7d9dd54e49b369522 Mon Sep 17 00:00:00 2001 From: Tuhin Sinha Date: Wed, 3 Feb 2016 20:11:45 -0800 Subject: [PATCH 11/11] working version with a new path_with_params function at the blob level to help with generation based queries of blobs. also added version-based blob copying and renaming. fixed a typo in _helpers.py. --- gcloud/storage/_helpers.py | 2 +- gcloud/storage/blob.py | 32 ++++++++++++++++- gcloud/storage/bucket.py | 70 +++++++++++++++++++++++--------------- 3 files changed, 75 insertions(+), 29 deletions(-) diff --git a/gcloud/storage/_helpers.py b/gcloud/storage/_helpers.py index e55fcf179a5f..7ca821fcda29 100644 --- a/gcloud/storage/_helpers.py +++ b/gcloud/storage/_helpers.py @@ -22,7 +22,7 @@ class _PropertyMixin(object): - """Abstract mixin for cloud storage classes with associated propertties. + """Abstract mixin for cloud storage classes with associated properties. Non-abstract subclasses should implement: - client diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 394702136520..be961405a338 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -61,12 +61,13 @@ class Blob(_PropertyMixin): _CHUNK_SIZE_MULTIPLE = 256 * 1024 """Number (256 KB, in bytes) that must divide the chunk size.""" - def __init__(self, name, bucket, chunk_size=None): + def __init__(self, name, bucket, chunk_size=None, generation=None): super(Blob, self).__init__(name=name) self.chunk_size = chunk_size # Check that setter accepts value. self.bucket = bucket self._acl = ObjectACL(self) + self.generation = generation @property def chunk_size(self): @@ -132,6 +133,22 @@ def path(self): return self.path_helper(self.bucket.path, self.name) + @property + def path_with_params(self): + """Getter property for the URL path to this Blob, with version. + + :rtype: tuple of ``path`` (a string) and ``params`` (a dictionary) + :returns: the URL path to this blob and a dictionary with the + generation that can be used in query_params for + connection.api_request + """ + + params = {} + if self.generation is not None: + params = {'generation': self.generation} + + return (self.path, params) + @property def client(self): """The client bound to this blob.""" @@ -638,6 +655,19 @@ def generation(self): if generation is not None: return int(generation) + @generation.setter + def generation(self, value): + """Set the generation for this blob. + + See: https://cloud.google.com/storage/docs/json_api/v1/objects + + :type value: integer or ``NoneType`` + :param value: the generation value for this blob. Setting this + value is useful when trying to retrieve specific + versions of a blob. + """ + self._patch_property('generation', value) + @property def id(self): """Retrieve the ID for the object. diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 277fa2eacbf6..9f2cb33e9ec4 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -103,7 +103,7 @@ def client(self): """The client bound to this bucket.""" return self._client - def blob(self, blob_name, chunk_size=None): + def blob(self, blob_name, chunk_size=None, generation=None): """Factory constructor for blob object. .. note:: @@ -118,10 +118,16 @@ def blob(self, blob_name, chunk_size=None): (1 MB). This must be a multiple of 256 KB per the API specification. + :type generation: integer + :param generation: The desired generation of the blob object. This + parameter affects generation based queries on + buckets that support versioning. + :rtype: :class:`gcloud.storage.blob.Blob` :returns: The blob object created. """ - return Blob(name=blob_name, bucket=self, chunk_size=chunk_size) + return Blob(name=blob_name, bucket=self, chunk_size=chunk_size, + generation=generation) def exists(self, client=None): """Determines whether or not this bucket exists. @@ -234,13 +240,11 @@ def get_blob(self, blob_name, client=None, generation=None): :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) - blob = Blob(bucket=self, name=blob_name) + blob = Blob(bucket=self, name=blob_name, generation=generation) + blob_path, query_params = blob.path_with_params try: - query_params = {} - if generation is not None: - query_params = {'generation': generation} response = client.connection.api_request( - method='GET', path=blob.path, + method='GET', path=blob_path, query_params=query_params, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) @@ -479,30 +483,34 @@ def copy_blob(self, blob, destination_bucket, new_name=None, :param versions: Optional. Copy each version. :rtype: :class:`gcloud.storage.blob.Blob` - :returns: The new Blob. + :returns: The new Blob if versions is ``False``, or will return + a list of new blob versions, and their old blob version + counterparts. """ client = self._require_client(client) if new_name is None: new_name = blob.name - new_blob = Blob(bucket=destination_bucket, name=new_name) - api_path = blob.path + '/copyTo' + new_blob.path - query_params = {} + tmp_blob = Blob(bucket=destination_bucket, name=new_name) + api_path = blob.path + '/copyTo' + tmp_blob.path + del tmp_blob # TODO(tsinha): Support multi-page results from list_blobs - if versions == True: - blob_versions = list(self.list_blobs(prefix=blob.name, versions=True, fields='name,generation,updated')) - for blob_version in blob_versions: - copy_result = client.connection.api_request( - method='POST', path=api_path, - query_params={'sourceGeneration': blob_version.generation}, - _target_object=new_blob) - else: + old_blobs = list(self.list_blobs(prefix=blob.name, versions=versions)) + new_blobs = [] + for old_blob in old_blobs: + new_blob = Blob(bucket=destination_bucket, name=new_name) copy_result = client.connection.api_request( - method='POST', path=api_path, _target_object=new_blob) - - new_blob._set_properties(copy_result) - return new_blob + method='POST', path=api_path, + query_params={'sourceGeneration': old_blob.generation}, + _target_object=new_blob) + new_blob._set_properties(copy_result) + new_blobs.append(new_blob) + + if versions: + return (new_blobs, old_blobs) + else: + return new_blobs[0] def rename_blob(self, blob, new_name, client=None): """Rename the given blob using copy and delete operations. @@ -527,11 +535,19 @@ def rename_blob(self, blob, new_name, client=None): to the ``client`` stored on the current bucket. :rtype: :class:`Blob` - :returns: The newly-renamed blob. + :returns: The newly-renamed blob if bucket versioning is off (or + there is only one version), otherwise will return blobs + for each newly-renamed version. """ - new_blob = self.copy_blob(blob, self, new_name, client=client, versions=True) - blob.delete(client=client) - return new_blob + new_blobs, old_blobs = self.copy_blob(blob, self, new_name, + client=client, versions=True) + for old_blob in old_blobs: + old_blob.delete(client=client) + + if len(new_blobs) == 1: + return new_blobs[0] + else: + return new_blobs @property def cors(self):