Verify MD5 is available

botocore/compat.py

@@ -237,3 +237,17 @@ def total_seconds(delta):
  day_in_seconds = delta.days * 24 * 3600.0
  micro_in_seconds = delta.microseconds / 10.0**6
  return day_in_seconds + delta.seconds + micro_in_seconds
+
+
+def md5_available():
+ """
+ Checks to see if md5 is available on this system. A given system might not
+ have access to it for various reasons, such as FIPS mode being enabled.
+ :return: True if md5 is available. False if not.
+ """
+ try:
+ import hashlib
+ hashlib.md5()
+ return True
+ except:
+ return False

botocore/exceptions.py

@@ -367,3 +367,7 @@ class InvalidConfigError(BotoCoreError):
 
 class RefreshWithMFAUnsupportedError(BotoCoreError):
  fmt = 'Cannot refresh credentials: MFA token required.'
+
+
+class MD5UnavailableError(BotoCoreError):
+ fmt = "This system does not support MD5 generation."

botocore/handlers.py

@@ -25,13 +25,13 @@
 import warnings
 
 from botocore.compat import urlsplit, urlunsplit, unquote, \
- json, quote, six, unquote_str, ensure_bytes
+ json, quote, six, unquote_str, ensure_bytes, md5_available
 from botocore.docs.utils import AutoPopulatedParam
 from botocore.docs.utils import HideParamFromOperations
 from botocore.docs.utils import AppendParamDocumentation
 from botocore.signers import add_generate_presigned_url
 from botocore.signers import add_generate_presigned_post
-from botocore.exceptions import ParamValidationError
+from botocore.exceptions import ParamValidationError, MD5UnavailableError
 from botocore.exceptions import UnsupportedTLSVersionWarning
 from botocore.utils import percent_encode, SAFE_CHARS
 
@@ -122,6 +122,9 @@ def json_decode_template_body(parsed, **kwargs):
 
 
 def calculate_md5(params, **kwargs):
+ if not md5_available():
+ raise MD5UnavailableError()
+
  request_dict = params
  if request_dict['body'] and 'Content-MD5' not in params['headers']:
  body = request_dict['body']
@@ -150,7 +153,7 @@ def _calculate_md5_from_file(fileobj):
 def conditionally_calculate_md5(params, **kwargs):
  """Only add a Content-MD5 when not using sigv4"""
  signer = kwargs['request_signer']
- if signer.signature_version not in ['v4', 's3v4']:
+ if signer.signature_version not in ['v4', 's3v4'] and md5_available():
  calculate_md5(params, **kwargs)
 
 
@@ -188,6 +191,9 @@ def copy_source_sse_md5(params, **kwargs):
 def _sse_md5(params, sse_member_prefix='SSECustomer'):
  if not _needs_s3_sse_customization(params, sse_member_prefix):
  return
+ if not md5_available():
+ raise MD5UnavailableError()
+
  sse_key_member = sse_member_prefix + 'Key'
  sse_md5_member = sse_member_prefix + 'KeyMD5'
  key_as_bytes = params[sse_key_member]

tests/unit/test_compat.py

@@ -12,8 +12,11 @@
 # language governing permissions and limitations under the License.
 
 import datetime
+import mock
 
-from botocore.compat import total_seconds, unquote_str, six, ensure_bytes
+from botocore.compat import (
+ total_seconds, unquote_str, six, ensure_bytes, md5_available
+)
 
 from tests import BaseEnvVar, unittest
 
@@ -78,3 +81,14 @@ def test_non_string_or_bytes_raises_error(self):
  value = 500
  with self.assertRaises(ValueError):
  ensure_bytes(value)
+
+
+class TestMD5Available(unittest.TestCase):
+ def test_available(self):
+ with mock.patch('hashlib.md5'):
+ self.assertTrue(md5_available())
+
+ def test_unavailable(self):
+ with mock.patch('hashlib.md5') as md5:
+ md5.side_effect = ValueError()
+ self.assertFalse(md5_available())

tests/unit/test_handlers.py

@@ -20,7 +20,7 @@
 
 import botocore
 import botocore.session
-from botocore.exceptions import ParamValidationError
+from botocore.exceptions import ParamValidationError, MD5UnavailableError
 from botocore.awsrequest import AWSRequest
 from botocore.compat import quote, six
 from botocore.model import OperationModel, ServiceModel
@@ -509,60 +509,7 @@ def test_switch_host_with_param(self):
  handlers.switch_host_with_param(request, 'PredictEndpoint')
  self.assertEqual(request.url, new_endpoint)
 
- def test_does_not_add_md5_when_v4(self):
- credentials = Credentials('key', 'secret')
- request_signer = RequestSigner(
- 's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
- request_dict = {'body': b'bar',
- 'url': 'https://s3.us-east-1.amazonaws.com',
- 'method': 'PUT',
- 'headers': {}}
- handlers.conditionally_calculate_md5(request_dict,
- request_signer=request_signer)
- self.assertTrue('Content-MD5' not in request_dict['headers'])
-
- def test_does_not_add_md5_when_s3v4(self):
- credentials = Credentials('key', 'secret')
- request_signer = RequestSigner(
- 's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
- request_dict = {'body': b'bar',
- 'url': 'https://s3.us-east-1.amazonaws.com',
- 'method': 'PUT',
- 'headers': {}}
- handlers.conditionally_calculate_md5(request_dict,
- request_signer=request_signer)
- self.assertTrue('Content-MD5' not in request_dict['headers'])
-
- def test_adds_md5_when_not_v4(self):
- credentials = Credentials('key', 'secret')
- request_signer = RequestSigner(
- 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
- request_dict = {'body': b'bar',
- 'url': 'https://s3.us-east-1.amazonaws.com',
- 'method': 'PUT',
- 'headers': {}}
- handlers.conditionally_calculate_md5(request_dict,
- request_signer=request_signer)
- self.assertTrue('Content-MD5' in request_dict['headers'])
 
- def test_adds_md5_with_file_like_body(self):
- request_dict = {
- 'body': six.BytesIO(b'foobar'),
- 'headers': {}
- }
- handlers.calculate_md5(request_dict)
- self.assertEqual(request_dict['headers']['Content-MD5'],
- 'OFj2IjCsPJFfMAxmQxLGPw==')
-
- def test_adds_md5_with_bytes_object(self):
- request_dict = {
- 'body': b'foobar',
- 'headers': {}
- }
- handlers.calculate_md5(request_dict)
- self.assertEqual(
- request_dict['headers']['Content-MD5'],
- 'OFj2IjCsPJFfMAxmQxLGPw==')
 
  def test_invalid_char_in_bucket_raises_exception(self):
  params = {
@@ -735,3 +682,127 @@ def test_s3_special_case_is_before_other_retry(self):
  self.assertTrue(s3_200_handler < general_retry_handler,
  "S3 200 error handler was supposed to be before "
  "the general retry handler, but it was not.")
+
+
+class BaseMD5Test(BaseSessionTest):
+ def setUp(self, **environ):
+ super(BaseMD5Test, self).setUp(**environ)
+ self.md5_object = mock.Mock()
+ self.md5_digest = mock.Mock(return_value=b'foo')
+ self.md5_object.digest = self.md5_digest
+ self.md5_builder = mock.Mock(return_value=self.md5_object)
+ self.md5_patch = mock.patch('hashlib.md5', self.md5_builder)
+ self.md5_patch.start()
+
+ def tearDown(self):
+ super(BaseMD5Test, self).tearDown()
+ self.md5_patch.stop()
+
+
+class TestSSEMD5(BaseMD5Test):
+ def test_sse_md5(self):
+ key = 'SSECustomerKey'
+ params = {key: b'foo'}
+ handlers.sse_md5(params)
+ self.assertEqual(params[key + 'MD5'], 'Zm9v')
+
+ key = 'CopySourceSSECustomerKey'
+ params = {key: b'foo'}
+ handlers.copy_source_sse_md5(params)
+ self.assertEqual(params[key + 'MD5'], 'Zm9v')
+
+ def test_raises_error_when_md5_unavailable(self):
+ self.md5_builder.side_effect = ValueError()
+
+ with self.assertRaises(MD5UnavailableError):
+ handlers.sse_md5({'SSECustomerKey': b'foo'})
+
+ with self.assertRaises(MD5UnavailableError):
+ handlers.copy_source_sse_md5({'CopySourceSSECustomerKey': b'foo'})
+
+
+class TestAddMD5(BaseMD5Test):
+ def test_does_not_add_md5_when_v4(self):
+ credentials = Credentials('key', 'secret')
+ request_signer = RequestSigner(
+ 's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
+ request_dict = {'body': b'bar',
+ 'url': 'https://s3.us-east-1.amazonaws.com',
+ 'method': 'PUT',
+ 'headers': {}}
+ handlers.conditionally_calculate_md5(request_dict,
+ request_signer=request_signer)
+ self.assertTrue('Content-MD5' not in request_dict['headers'])
+
+ def test_does_not_add_md5_when_s3v4(self):
+ credentials = Credentials('key', 'secret')
+ request_signer = RequestSigner(
+ 's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
+ request_dict = {'body': b'bar',
+ 'url': 'https://s3.us-east-1.amazonaws.com',
+ 'method': 'PUT',
+ 'headers': {}}
+ handlers.conditionally_calculate_md5(request_dict,
+ request_signer=request_signer)
+ self.assertTrue('Content-MD5' not in request_dict['headers'])
+
+ def test_conditional_does_not_add_when_md5_unavailable(self):
+ credentials = Credentials('key', 'secret')
+ request_signer = RequestSigner(
+ 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+ request_dict = {'body': b'bar',
+ 'url': 'https://s3.us-east-1.amazonaws.com',
+ 'method': 'PUT',
+ 'headers': {}}
+
+ self.md5_builder.side_effect = ValueError()
+ handlers.conditionally_calculate_md5(
+ request_dict, request_signer=request_signer)
+ self.assertFalse('Content-MD5' in request_dict['headers'])
+
+ def test_add_md5_raises_error_when_md5_unavailable(self):
+ credentials = Credentials('key', 'secret')
+ request_signer = RequestSigner(
+ 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+ request_dict = {'body': b'bar',
+ 'url': 'https://s3.us-east-1.amazonaws.com',
+ 'method': 'PUT',
+ 'headers': {}}
+
+ self.md5_builder.side_effect = ValueError()
+ with self.assertRaises(MD5UnavailableError):
+ handlers.calculate_md5(
+ request_dict, request_signer=request_signer)
+
+ def test_adds_md5_when_not_v4(self):
+ credentials = Credentials('key', 'secret')
+ request_signer = RequestSigner(
+ 's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+ request_dict = {'body': b'bar',
+ 'url': 'https://s3.us-east-1.amazonaws.com',
+ 'method': 'PUT',
+ 'headers': {}}
+ handlers.conditionally_calculate_md5(request_dict,
+ request_signer=request_signer)
+ self.assertTrue('Content-MD5' in request_dict['headers'])
+
+ def test_add_md5_with_file_like_body(self):
+ request_dict = {
+ 'body': six.BytesIO(b'foobar'),
+ 'headers': {}
+ }
+ self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
+ handlers.calculate_md5(request_dict)
+ self.assertEqual(request_dict['headers']['Content-MD5'],
+ 'OFj2IjCsPJFfMAxmQxLGPw==')
+
+ def test_add_md5_with_bytes_object(self):
+ request_dict = {
+ 'body': b'foobar',
+ 'headers': {}
+ }
+ self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
+ handlers.calculate_md5(request_dict)
+ self.assertEqual(
+ request_dict['headers']['Content-MD5'],
+ 'OFj2IjCsPJFfMAxmQxLGPw==')