Skip to content

Commit

Permalink
perf(storage): blob downloads gzipped content.
Browse files Browse the repository at this point in the history
GET requests now contain the header accept-encoding:gzip
This improves performance of compressible strings which
were uploaded as gzips.

The caller is not required to do any decompression
because decompressiong is handle by the library.

Confusing `IOError: CRC check failed` exceptions will be
risen in the case of mid-download connection breaks.
  • Loading branch information
tartavull committed May 17, 2017
1 parent 937165e commit a7a8390
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
1 change: 1 addition & 0 deletions storage/google/cloud/storage/blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,7 @@ def download_to_file(self, file_obj, client=None):
"""
download_url = self._get_download_url()
headers = _get_encryption_headers(self._encryption_key)
headers['accept-encoding'] = 'gzip'
transport = self._make_transport(client)

try:
Expand Down
8 changes: 6 additions & 2 deletions storage/tests/unit/test_blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,7 @@ def _check_session_mocks(self, client, fake_session_factory,
# **MUTABLE** headers and it was mutated before the
# second request.
headers['range'] = 'bytes=3-5'
headers['accept-encoding'] = 'gzip'
call = mock.call(
'GET', expected_url, data=None, headers=headers)
self.assertEqual(fake_transport.request.mock_calls, [call, call])
Expand Down Expand Up @@ -493,7 +494,8 @@ def test_download_to_file_with_failure(self, fake_session_factory):
fake_session_factory.assert_called_once_with(client._credentials)
# Check that the transport was called once.
transport.request.assert_called_once_with(
'GET', blob.media_link, data=None, headers={})
'GET', blob.media_link, data=None,
headers={'accept-encoding':'gzip'})

@mock.patch('google.auth.transport.requests.AuthorizedSession')
def test_download_to_file_wo_media_link(self, fake_session_factory):
Expand Down Expand Up @@ -553,7 +555,8 @@ def _download_to_file_helper(self, fake_session_factory, use_chunks=False):
# Check that exactly one transport was created.
fake_session_factory.assert_called_once_with(client._credentials)
fake_transport.request.assert_called_once_with(
'GET', media_link, data=None, headers={})
'GET', media_link, data=None,
headers={'accept-encoding': 'gzip'})

def test_download_to_file_default(self):
self._download_to_file_helper()
Expand Down Expand Up @@ -645,6 +648,7 @@ def test_download_to_filename_w_key(self, fake_session_factory):
'X-Goog-Encryption-Key-Sha256': header_key_hash_value,
'X-Goog-Encryption-Algorithm': 'AES256',
'X-Goog-Encryption-Key': header_key_value,
'accept-encoding':'gzip'
}
self._check_session_mocks(
client, fake_session_factory, media_link, headers=key_headers)
Expand Down

0 comments on commit a7a8390

Please sign in to comment.