diff --git a/botocore/compat.py b/botocore/compat.py index 936589c63f..028d02d65f 100644 --- a/botocore/compat.py +++ b/botocore/compat.py @@ -16,10 +16,15 @@ import sys import inspect import warnings +import hashlib +import logging from botocore.vendored import six +from botocore.exceptions import MD5UnavailableError from botocore.vendored.requests.packages.urllib3 import exceptions +logger = logging.getLogger(__name__) + if six.PY3: from six.moves import http_client @@ -237,3 +242,30 @@ def total_seconds(delta): day_in_seconds = delta.days * 24 * 3600.0 micro_in_seconds = delta.microseconds / 10.0**6 return day_in_seconds + delta.seconds + micro_in_seconds + + +# Checks to see if md5 is available on this system. A given system might not +# have access to it for various reasons, such as FIPS mode being enabled. +try: + hashlib.md5() + MD5_AVAILABLE = True +except ValueError: + MD5_AVAILABLE = False + + +def get_md5(*args, **kwargs): + """ + Attempts to get an md5 hashing object. + + :param raise_error_if_unavailable: raise an error if md5 is unavailable on + this system. If False, None will be returned if it is unavailable. + :type raise_error_if_unavailable: bool + :param args: Args to pass to the MD5 constructor + :param kwargs: Key word arguments to pass to the MD5 constructor + :return: An MD5 hashing object if available. If it is unavailable, None + is returned if raise_error_if_unavailable is set to False. + """ + if MD5_AVAILABLE: + return hashlib.md5(*args, **kwargs) + else: + raise MD5UnavailableError() diff --git a/botocore/exceptions.py b/botocore/exceptions.py index 7ccc89265a..c4f1404c9e 100644 --- a/botocore/exceptions.py +++ b/botocore/exceptions.py @@ -367,3 +367,7 @@ class InvalidConfigError(BotoCoreError): class RefreshWithMFAUnsupportedError(BotoCoreError): fmt = 'Cannot refresh credentials: MFA token required.' + + +class MD5UnavailableError(BotoCoreError): + fmt = "This system does not support MD5 generation." diff --git a/botocore/handlers.py b/botocore/handlers.py index 1b8f07b9af..7a384e64ff 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -17,7 +17,6 @@ """ import base64 -import hashlib import logging import xml.etree.cElementTree import copy @@ -25,7 +24,7 @@ import warnings from botocore.compat import urlsplit, urlunsplit, unquote, \ - json, quote, six, unquote_str, ensure_bytes + json, quote, six, unquote_str, ensure_bytes, get_md5, MD5_AVAILABLE from botocore.docs.utils import AutoPopulatedParam from botocore.docs.utils import HideParamFromOperations from botocore.docs.utils import AppendParamDocumentation @@ -134,13 +133,13 @@ def calculate_md5(params, **kwargs): def _calculate_md5_from_bytes(body_bytes): - md5 = hashlib.md5(body_bytes) + md5 = get_md5(body_bytes) return md5.digest() def _calculate_md5_from_file(fileobj): start_position = fileobj.tell() - md5 = hashlib.md5() + md5 = get_md5() for chunk in iter(lambda: fileobj.read(1024 * 1024), b''): md5.update(chunk) fileobj.seek(start_position) @@ -150,7 +149,7 @@ def _calculate_md5_from_file(fileobj): def conditionally_calculate_md5(params, **kwargs): """Only add a Content-MD5 when not using sigv4""" signer = kwargs['request_signer'] - if signer.signature_version not in ['v4', 's3v4']: + if signer.signature_version not in ['v4', 's3v4'] and MD5_AVAILABLE: calculate_md5(params, **kwargs) @@ -188,13 +187,14 @@ def copy_source_sse_md5(params, **kwargs): def _sse_md5(params, sse_member_prefix='SSECustomer'): if not _needs_s3_sse_customization(params, sse_member_prefix): return + sse_key_member = sse_member_prefix + 'Key' sse_md5_member = sse_member_prefix + 'KeyMD5' key_as_bytes = params[sse_key_member] if isinstance(key_as_bytes, six.text_type): key_as_bytes = key_as_bytes.encode('utf-8') key_md5_str = base64.b64encode( - hashlib.md5(key_as_bytes).digest()).decode('utf-8') + get_md5(key_as_bytes).digest()).decode('utf-8') key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8') params[sse_key_member] = key_b64_encoded params[sse_md5_member] = key_md5_str diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py index fbaae33690..1683ed5b21 100644 --- a/tests/unit/test_compat.py +++ b/tests/unit/test_compat.py @@ -12,8 +12,12 @@ # language governing permissions and limitations under the License. import datetime +import mock -from botocore.compat import total_seconds, unquote_str, six, ensure_bytes +from botocore.exceptions import MD5UnavailableError +from botocore.compat import ( + total_seconds, unquote_str, six, ensure_bytes, get_md5 +) from tests import BaseEnvVar, unittest @@ -78,3 +82,16 @@ def test_non_string_or_bytes_raises_error(self): value = 500 with self.assertRaises(ValueError): ensure_bytes(value) + + +class TestGetMD5(unittest.TestCase): + def test_available(self): + md5 = mock.Mock() + with mock.patch('botocore.compat.MD5_AVAILABLE', True): + with mock.patch('hashlib.md5', mock.Mock(return_value=md5)): + self.assertEqual(get_md5(), md5) + + def test_unavailable_raises_error(self): + with mock.patch('botocore.compat.MD5_AVAILABLE', False): + with self.assertRaises(MD5UnavailableError): + get_md5() diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index bcbbf76680..9d56765e2b 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -20,7 +20,7 @@ import botocore import botocore.session -from botocore.exceptions import ParamValidationError +from botocore.exceptions import ParamValidationError, MD5UnavailableError from botocore.awsrequest import AWSRequest from botocore.compat import quote, six from botocore.model import OperationModel, ServiceModel @@ -199,45 +199,6 @@ def test_500_response_can_be_none(self): # object is None. We need to handle this case. handlers.check_for_200_error(None) - def test_sse_params(self): - for op in ('HeadObject', 'GetObject', 'PutObject', 'CopyObject', - 'CreateMultipartUpload', 'UploadPart', 'UploadPartCopy'): - event = 'before-parameter-build.s3.%s' % op - params = {'SSECustomerKey': b'bar', - 'SSECustomerAlgorithm': 'AES256'} - self.session.emit(event, params=params, model=mock.Mock()) - self.assertEqual(params['SSECustomerKey'], 'YmFy') - self.assertEqual(params['SSECustomerKeyMD5'], - 'N7UdGUp1E+RbVvZSTy1R8g==') - - def test_sse_params_as_str(self): - event = 'before-parameter-build.s3.PutObject' - params = {'SSECustomerKey': 'bar', - 'SSECustomerAlgorithm': 'AES256'} - self.session.emit(event, params=params, model=mock.Mock()) - self.assertEqual(params['SSECustomerKey'], 'YmFy') - self.assertEqual(params['SSECustomerKeyMD5'], - 'N7UdGUp1E+RbVvZSTy1R8g==') - - def test_copy_source_sse_params(self): - for op in ['CopyObject', 'UploadPartCopy']: - event = 'before-parameter-build.s3.%s' % op - params = {'CopySourceSSECustomerKey': b'bar', - 'CopySourceSSECustomerAlgorithm': 'AES256'} - self.session.emit(event, params=params, model=mock.Mock()) - self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy') - self.assertEqual(params['CopySourceSSECustomerKeyMD5'], - 'N7UdGUp1E+RbVvZSTy1R8g==') - - def test_copy_source_sse_params_as_str(self): - event = 'before-parameter-build.s3.CopyObject' - params = {'CopySourceSSECustomerKey': 'bar', - 'CopySourceSSECustomerAlgorithm': 'AES256'} - self.session.emit(event, params=params, model=mock.Mock()) - self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy') - self.assertEqual(params['CopySourceSSECustomerKeyMD5'], - 'N7UdGUp1E+RbVvZSTy1R8g==') - def test_route53_resource_id(self): event = 'before-parameter-build.route53.GetHostedZone' params = {'Id': '/hostedzone/ABC123', @@ -509,60 +470,7 @@ def test_switch_host_with_param(self): handlers.switch_host_with_param(request, 'PredictEndpoint') self.assertEqual(request.url, new_endpoint) - def test_does_not_add_md5_when_v4(self): - credentials = Credentials('key', 'secret') - request_signer = RequestSigner( - 's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock()) - request_dict = {'body': b'bar', - 'url': 'https://s3.us-east-1.amazonaws.com', - 'method': 'PUT', - 'headers': {}} - handlers.conditionally_calculate_md5(request_dict, - request_signer=request_signer) - self.assertTrue('Content-MD5' not in request_dict['headers']) - def test_does_not_add_md5_when_s3v4(self): - credentials = Credentials('key', 'secret') - request_signer = RequestSigner( - 's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock()) - request_dict = {'body': b'bar', - 'url': 'https://s3.us-east-1.amazonaws.com', - 'method': 'PUT', - 'headers': {}} - handlers.conditionally_calculate_md5(request_dict, - request_signer=request_signer) - self.assertTrue('Content-MD5' not in request_dict['headers']) - - def test_adds_md5_when_not_v4(self): - credentials = Credentials('key', 'secret') - request_signer = RequestSigner( - 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock()) - request_dict = {'body': b'bar', - 'url': 'https://s3.us-east-1.amazonaws.com', - 'method': 'PUT', - 'headers': {}} - handlers.conditionally_calculate_md5(request_dict, - request_signer=request_signer) - self.assertTrue('Content-MD5' in request_dict['headers']) - - def test_adds_md5_with_file_like_body(self): - request_dict = { - 'body': six.BytesIO(b'foobar'), - 'headers': {} - } - handlers.calculate_md5(request_dict) - self.assertEqual(request_dict['headers']['Content-MD5'], - 'OFj2IjCsPJFfMAxmQxLGPw==') - - def test_adds_md5_with_bytes_object(self): - request_dict = { - 'body': b'foobar', - 'headers': {} - } - handlers.calculate_md5(request_dict) - self.assertEqual( - request_dict['headers']['Content-MD5'], - 'OFj2IjCsPJFfMAxmQxLGPw==') def test_invalid_char_in_bucket_raises_exception(self): params = { @@ -735,3 +643,164 @@ def test_s3_special_case_is_before_other_retry(self): self.assertTrue(s3_200_handler < general_retry_handler, "S3 200 error handler was supposed to be before " "the general retry handler, but it was not.") + + +class BaseMD5Test(BaseSessionTest): + def setUp(self, **environ): + super(BaseMD5Test, self).setUp(**environ) + self.md5_object = mock.Mock() + self.md5_digest = mock.Mock(return_value=b'foo') + self.md5_object.digest = self.md5_digest + md5_builder = mock.Mock(return_value=self.md5_object) + self.md5_patch = mock.patch('hashlib.md5', md5_builder) + self.md5_patch.start() + self._md5_available_patch = None + self.set_md5_available() + + def tearDown(self): + super(BaseMD5Test, self).tearDown() + self.md5_patch.stop() + if self._md5_available_patch: + self._md5_available_patch.stop() + + def set_md5_available(self, is_available=True): + if self._md5_available_patch: + self._md5_available_patch.stop() + + self._md5_available_patch = \ + mock.patch('botocore.compat.MD5_AVAILABLE', is_available) + self._md5_available_patch.start() + + +class TestSSEMD5(BaseMD5Test): + def test_raises_error_when_md5_unavailable(self): + self.set_md5_available(False) + + with self.assertRaises(MD5UnavailableError): + handlers.sse_md5({'SSECustomerKey': b'foo'}) + + with self.assertRaises(MD5UnavailableError): + handlers.copy_source_sse_md5({'CopySourceSSECustomerKey': b'foo'}) + + def test_sse_params(self): + for op in ('HeadObject', 'GetObject', 'PutObject', 'CopyObject', + 'CreateMultipartUpload', 'UploadPart', 'UploadPartCopy'): + event = 'before-parameter-build.s3.%s' % op + params = {'SSECustomerKey': b'bar', + 'SSECustomerAlgorithm': 'AES256'} + self.session.emit(event, params=params, model=mock.Mock()) + self.assertEqual(params['SSECustomerKey'], 'YmFy') + self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v') + + def test_sse_params_as_str(self): + event = 'before-parameter-build.s3.PutObject' + params = {'SSECustomerKey': 'bar', + 'SSECustomerAlgorithm': 'AES256'} + self.session.emit(event, params=params, model=mock.Mock()) + self.assertEqual(params['SSECustomerKey'], 'YmFy') + self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v') + + def test_copy_source_sse_params(self): + for op in ['CopyObject', 'UploadPartCopy']: + event = 'before-parameter-build.s3.%s' % op + params = {'CopySourceSSECustomerKey': b'bar', + 'CopySourceSSECustomerAlgorithm': 'AES256'} + self.session.emit(event, params=params, model=mock.Mock()) + self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy') + self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v') + + def test_copy_source_sse_params_as_str(self): + event = 'before-parameter-build.s3.CopyObject' + params = {'CopySourceSSECustomerKey': 'bar', + 'CopySourceSSECustomerAlgorithm': 'AES256'} + self.session.emit(event, params=params, model=mock.Mock()) + self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy') + self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v') + + +class TestAddMD5(BaseMD5Test): + def test_does_not_add_md5_when_v4(self): + credentials = Credentials('key', 'secret') + request_signer = RequestSigner( + 's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock()) + request_dict = {'body': b'bar', + 'url': 'https://s3.us-east-1.amazonaws.com', + 'method': 'PUT', + 'headers': {}} + handlers.conditionally_calculate_md5(request_dict, + request_signer=request_signer) + self.assertTrue('Content-MD5' not in request_dict['headers']) + + def test_does_not_add_md5_when_s3v4(self): + credentials = Credentials('key', 'secret') + request_signer = RequestSigner( + 's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock()) + request_dict = {'body': b'bar', + 'url': 'https://s3.us-east-1.amazonaws.com', + 'method': 'PUT', + 'headers': {}} + handlers.conditionally_calculate_md5(request_dict, + request_signer=request_signer) + self.assertTrue('Content-MD5' not in request_dict['headers']) + + def test_conditional_does_not_add_when_md5_unavailable(self): + credentials = Credentials('key', 'secret') + request_signer = RequestSigner( + 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock()) + request_dict = {'body': b'bar', + 'url': 'https://s3.us-east-1.amazonaws.com', + 'method': 'PUT', + 'headers': {}} + + self.set_md5_available(False) + with mock.patch('botocore.handlers.MD5_AVAILABLE', False): + handlers.conditionally_calculate_md5( + request_dict, request_signer=request_signer) + self.assertFalse('Content-MD5' in request_dict['headers']) + + def test_add_md5_raises_error_when_md5_unavailable(self): + credentials = Credentials('key', 'secret') + request_signer = RequestSigner( + 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock()) + request_dict = {'body': b'bar', + 'url': 'https://s3.us-east-1.amazonaws.com', + 'method': 'PUT', + 'headers': {}} + + self.set_md5_available(False) + with self.assertRaises(MD5UnavailableError): + handlers.calculate_md5( + request_dict, request_signer=request_signer) + + def test_adds_md5_when_not_v4(self): + credentials = Credentials('key', 'secret') + request_signer = RequestSigner( + 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock()) + request_dict = {'body': b'bar', + 'url': 'https://s3.us-east-1.amazonaws.com', + 'method': 'PUT', + 'headers': {}} + handlers.conditionally_calculate_md5(request_dict, + request_signer=request_signer) + self.assertTrue('Content-MD5' in request_dict['headers']) + + def test_add_md5_with_file_like_body(self): + request_dict = { + 'body': six.BytesIO(b'foobar'), + 'headers': {} + } + self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?' + handlers.calculate_md5(request_dict) + self.assertEqual(request_dict['headers']['Content-MD5'], + 'OFj2IjCsPJFfMAxmQxLGPw==') + + def test_add_md5_with_bytes_object(self): + request_dict = { + 'body': b'foobar', + 'headers': {} + } + self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?' + handlers.calculate_md5(request_dict) + self.assertEqual( + request_dict['headers']['Content-MD5'], + 'OFj2IjCsPJFfMAxmQxLGPw==')