diff --git a/gcloud/storage/_helpers.py b/gcloud/storage/_helpers.py index e55fcf179a5f..7ca821fcda29 100644 --- a/gcloud/storage/_helpers.py +++ b/gcloud/storage/_helpers.py @@ -22,7 +22,7 @@ class _PropertyMixin(object): - """Abstract mixin for cloud storage classes with associated propertties. + """Abstract mixin for cloud storage classes with associated properties. Non-abstract subclasses should implement: - client diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 6b1280a31289..9351bd218cda 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -67,12 +67,13 @@ class Blob(_PropertyMixin): _CHUNK_SIZE_MULTIPLE = 256 * 1024 """Number (256 KB, in bytes) that must divide the chunk size.""" - def __init__(self, name, bucket, chunk_size=None): + def __init__(self, name, bucket, chunk_size=None, generation=None): super(Blob, self).__init__(name=name) self.chunk_size = chunk_size # Check that setter accepts value. self.bucket = bucket self._acl = ObjectACL(self) + self.generation = generation @property def chunk_size(self): @@ -138,6 +139,22 @@ def path(self): return self.path_helper(self.bucket.path, self.name) + @property + def path_with_params(self): + """Getter property for the URL path to this Blob, with version. + + :rtype: tuple of ``path`` (a string) and ``params`` (a dictionary) + :returns: the URL path to this blob and a dictionary with the + generation that can be used in query_params for + connection.api_request + """ + + params = {} + if self.generation is not None: + params = {'generation': self.generation} + + return (self.path, params) + @property def client(self): """The client bound to this blob.""" @@ -253,6 +270,9 @@ def exists(self, client=None): # We only need the status code (200 or not) so we seek to # minimize the returned payload. query_params = {'fields': 'name'} + if self.generation is not None: + query_params['generation'] = self.generation + # We intentionally pass `_target_object=None` since fields=name # would limit the local properties. client.connection.api_request(method='GET', path=self.path, @@ -278,7 +298,8 @@ def delete(self, client=None): (propagated from :meth:`gcloud.storage.bucket.Bucket.delete_blob`). """ - return self.bucket.delete_blob(self.name, client=client) + return self.bucket.delete_blob(self.name, client=client, + generation=self.generation) def download_to_file(self, file_obj, encryption_key=None, client=None): """Download the contents of this blob into a file-like object. @@ -743,6 +764,19 @@ def generation(self): if generation is not None: return int(generation) + @generation.setter + def generation(self, value): + """Set the generation for this blob. + + See: https://cloud.google.com/storage/docs/json_api/v1/objects + + :type value: integer or ``NoneType`` + :param value: the generation value for this blob. Setting this + value is useful when trying to retrieve specific + versions of a blob. + """ + self._patch_property('generation', value) + @property def id(self): """Retrieve the ID for the object. diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index d4e82b61bdb5..c96b044783c6 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -103,7 +103,7 @@ def client(self): """The client bound to this bucket.""" return self._client - def blob(self, blob_name, chunk_size=None): + def blob(self, blob_name, chunk_size=None, generation=None): """Factory constructor for blob object. .. note:: @@ -118,10 +118,16 @@ def blob(self, blob_name, chunk_size=None): (1 MB). This must be a multiple of 256 KB per the API specification. + :type generation: integer + :param generation: The desired generation of the blob object. This + parameter affects generation based queries on + buckets that support versioning. + :rtype: :class:`gcloud.storage.blob.Blob` :returns: The blob object created. """ - return Blob(name=blob_name, bucket=self, chunk_size=chunk_size) + return Blob(name=blob_name, bucket=self, chunk_size=chunk_size, + generation=generation) def exists(self, client=None): """Determines whether or not this bucket exists. @@ -205,7 +211,7 @@ def path(self): return self.path_helper(self.name) - def get_blob(self, blob_name, client=None): + def get_blob(self, blob_name, client=None, generation=None): """Get a blob object by name. This will return None if the blob doesn't exist:: @@ -217,6 +223,10 @@ def get_blob(self, blob_name, client=None): >>> print bucket.get_blob('/does-not-exist.txt') None + >>> print bucket.get_blob( + ... '/path/to/versioned_blob.txt', + ... generation=generation_id) + :type blob_name: string :param blob_name: The name of the blob to retrieve. @@ -225,14 +235,20 @@ def get_blob(self, blob_name, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type generation: int + :param generation: Optional. The generation id to retrieve in a bucket + that supports versioning. + :rtype: :class:`gcloud.storage.blob.Blob` or None :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) - blob = Blob(bucket=self, name=blob_name) + blob = Blob(bucket=self, name=blob_name, generation=generation) + blob_path, query_params = blob.path_with_params try: response = client.connection.api_request( - method='GET', path=blob.path, _target_object=blob) + method='GET', path=blob_path, + query_params=query_params, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when @@ -360,7 +376,7 @@ def delete(self, force=False, client=None): client.connection.api_request(method='DELETE', path=self.path, _target_object=None) - def delete_blob(self, blob_name, client=None): + def delete_blob(self, blob_name, client=None, generation=None): """Deletes a blob from the current bucket. If the blob isn't found (backend 404), raises a @@ -387,6 +403,10 @@ def delete_blob(self, blob_name, client=None): :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type generation: int + :param generation: Optional. The generation of this object to delete. + Only works on buckets with versioning enabled. + :raises: :class:`gcloud.exceptions.NotFound` (to suppress the exception, call ``delete_blobs``, passing a no-op ``on_error`` callback, e.g.:: @@ -395,10 +415,15 @@ def delete_blob(self, blob_name, client=None): """ client = self._require_client(client) blob_path = Blob.path_helper(self.path, blob_name) + query_params = {} + if generation is not None: + query_params = {'generation': generation} + # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or # in a batch request). client.connection.api_request(method='DELETE', path=blob_path, + query_params=query_params, _target_object=None) def delete_blobs(self, blobs, on_error=None, client=None): @@ -426,7 +451,13 @@ def delete_blobs(self, blobs, on_error=None, client=None): blob_name = blob if not isinstance(blob_name, six.string_types): blob_name = blob.name - self.delete_blob(blob_name, client=client) + + generation = None + if hasattr(blob, 'generation'): + generation = blob.generation + + self.delete_blob(blob_name, client=client, + generation=generation) except NotFound: if on_error is not None: on_error(blob) @@ -434,7 +465,7 @@ def delete_blobs(self, blobs, on_error=None, client=None): raise def copy_blob(self, blob, destination_bucket, new_name=None, - client=None): + client=None, versions=False): """Copy the given blob to the given bucket, optionally with a new name. :type blob: :class:`gcloud.storage.blob.Blob` @@ -451,18 +482,38 @@ def copy_blob(self, blob, destination_bucket, new_name=None, :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type versions: boolean + :param versions: Optional. Copy each version. + :rtype: :class:`gcloud.storage.blob.Blob` - :returns: The new Blob. + :returns: The new Blob if versions is ``False``, or will return + a list of new blob versions, and their old blob version + counterparts. """ client = self._require_client(client) if new_name is None: new_name = blob.name - new_blob = Blob(bucket=destination_bucket, name=new_name) - api_path = blob.path + '/copyTo' + new_blob.path - copy_result = client.connection.api_request( - method='POST', path=api_path, _target_object=new_blob) - new_blob._set_properties(copy_result) - return new_blob + + tmp_blob = Blob(bucket=destination_bucket, name=new_name) + api_path = blob.path + '/copyTo' + tmp_blob.path + del tmp_blob + + # TODO(tsinha): Support multi-page results from list_blobs + old_blobs = list(self.list_blobs(prefix=blob.name, versions=versions)) + new_blobs = [] + for old_blob in old_blobs: + new_blob = Blob(bucket=destination_bucket, name=new_name) + copy_result = client.connection.api_request( + method='POST', path=api_path, + query_params={'sourceGeneration': old_blob.generation}, + _target_object=new_blob) + new_blob._set_properties(copy_result) + new_blobs.append(new_blob) + + if versions: + return (new_blobs, old_blobs) + else: + return new_blobs[0] def rename_blob(self, blob, new_name, client=None): """Rename the given blob using copy and delete operations. @@ -487,11 +538,19 @@ def rename_blob(self, blob, new_name, client=None): to the ``client`` stored on the current bucket. :rtype: :class:`Blob` - :returns: The newly-renamed blob. - """ - new_blob = self.copy_blob(blob, self, new_name, client=client) - blob.delete(client=client) - return new_blob + :returns: The newly-renamed blob if bucket versioning is off (or + there is only one version), otherwise will return blobs + for each newly-renamed version. + """ + new_blobs, old_blobs = self.copy_blob(blob, self, new_name, + client=client, versions=True) + for old_blob in old_blobs: + old_blob.delete(client=client) + + if len(new_blobs) == 1: + return new_blobs[0] + else: + return new_blobs @property def cors(self): diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index 3ae92ed76a3d..44de99b53f49 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -278,6 +278,38 @@ def test_exists_hit(self): bucket._blobs[BLOB_NAME] = 1 self.assertTrue(blob.exists()) + def test_exists_hit_w_generation(self): + from six.moves.http_client import OK + BLOB_NAME = 'blob-name' + GENERATION = 999 + found_response = {'status': OK} + connection = _Connection(found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + self.assertTrue(blob.exists()) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['query_params']['generation'], GENERATION) + + def test_exists_hit_w_none_generation(self): + from six.moves.http_client import OK + BLOB_NAME = 'blob-name' + GENERATION = None + found_response = {'status': OK} + connection = _Connection(found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + self.assertTrue(blob.exists()) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertNotIn('generation', kw['query_params']) + def test_delete(self): from six.moves.http_client import NOT_FOUND BLOB_NAME = 'blob-name' @@ -291,6 +323,42 @@ def test_delete(self): self.assertFalse(blob.exists()) self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + def test_delete_w_generation(self): + from six.moves.http_client import NOT_FOUND + BLOB_NAME = 'blob-name' + GENERATION = 999 + not_found_response = {'status': NOT_FOUND} + connection = _Connection(not_found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + blob.delete() + self.assertFalse(blob.exists()) + self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['query_params']['generation'], GENERATION) + + def test_delete_w_none_generation(self): + from six.moves.http_client import NOT_FOUND + BLOB_NAME = 'blob-name' + GENERATION = None + not_found_response = {'status': NOT_FOUND} + connection = _Connection(not_found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties={'generation': GENERATION}) + bucket._blobs[BLOB_NAME] = 1 + blob.delete() + self.assertFalse(blob.exists()) + self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertNotIn('generation', kw['query_params']) + def test_download_to_file_wo_media_link(self): from six.moves.http_client import OK from six.moves.http_client import PARTIAL_CONTENT @@ -1372,7 +1440,8 @@ def __init__(self, client=None): self._copied = [] self._deleted = [] - def delete_blob(self, blob_name, client=None): + def delete_blob(self, blob_name, client=None, + generation=None): # pylint: disable=W0613 del self._blobs[blob_name] self._deleted.append((blob_name, client)) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index b22e51543a45..e42dec2cfbf8 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -277,6 +277,48 @@ def test_get_blob_hit(self): self.assertEqual(kw['method'], 'GET') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + def test_get_blob_miss_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-w-version' + GENERATION = 999 + connection = _Connection({'name': BLOB_NAME, 'generation': 1}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob, None) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['query_params'], {'generation': GENERATION}) + + def test_get_blob_hit_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-w-version' + GENERATION = 999 + connection = _Connection({'name': BLOB_NAME, 'generation': GENERATION}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob.bucket is bucket) + self.assertEqual(blob.name, BLOB_NAME) + self.assertEqual(blob.generation, GENERATION) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['query_params'], {'generation': GENERATION}) + + def test_get_blob_hit_w_none_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name-wo-version' + GENERATION = None + connection = _Connection({'name': BLOB_NAME}) + client = _Client(connection) + bucket = self._makeOne(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) + self.assertTrue(blob.bucket is bucket) + self.assertEqual(blob.name, BLOB_NAME) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertNotIn('generation', kw['query_params']) + def test_list_blobs_defaults(self): NAME = 'name' connection = _Connection({'items': []}) @@ -458,6 +500,46 @@ def test_delete_blob_hit(self): self.assertEqual(kw['method'], 'DELETE') self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) + def test_delete_blob_miss_w_generation(self): + from gcloud.exceptions import NotFound + NAME = 'name' + BLOB_NAME = 'blob_name' + GENERATION = 1 + connection = _Connection() + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + self.assertRaises(NotFound, bucket.delete_blob, + BLOB_NAME, generation=GENERATION) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertEqual(kw['query_params'], {'generation': GENERATION}) + + def test_delete_blob_hit_w_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name' + GENERATION = 999 + connection = _Connection({}) + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) + self.assertTrue(result is None) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertEqual(kw['query_params'], {'generation': GENERATION}) + + def test_delete_blob_hit_w_none_generation(self): + NAME = 'name' + BLOB_NAME = 'blob-name' + GENERATION = None + connection = _Connection({}) + client = _Client(connection) + bucket = self._makeOne(client=client, name=NAME) + result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) + self.assertTrue(result is None) + kw, = connection._requested + self.assertEqual(kw['method'], 'DELETE') + self.assertNotIn('generation', kw['query_params']) + def test_delete_blobs_empty(self): NAME = 'name' connection = _Connection()