Merge customizations for S3

docs/source/guide/configuration.rst

@@ -236,6 +236,15 @@ You can set configuration settings using system-wide environment variables. Thes
     A comma-delimited list of regions to sign when signing with SigV4a.  For more
     information, see the ``sigv4a_signing_region_set`` configuration file section.
 
+``AWS_REQUEST_CHECKSUM_CALCULATION``
+    Determines when a checksum will be calculated for request payloads. For more
+    information, see the ``request_checksum_calculation`` configuration file section.
+
+
+``AWS_RESPONSE_CHECKSUM_VALIDATION``
+    Determines when checksum validation will be performed on response payloads. For more
+    information, see the ``response_checksum_validation`` configuration file section.
+
 
 Using a configuration file
 --------------------------
@@ -476,6 +485,31 @@ in the ``~/.aws/config`` file.
     the SDK will check if the service has modeled a default; if none is found, this will
     default to ``*``.
 
+``request_checksum_calculation``
+    Determines when a checksum will be calculated for request payloads. Valid values are:
+
+    * ``when_supported`` -- When set, a checksum will be calculated for
+      all request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or a
+      ``requestAlgorithmMember`` is modeled.
+
+    * ``when_required`` -- When set, a checksum will only be calculated
+      for request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or where a
+      ``requestAlgorithmMember`` is modeled and supplied.
+
+``response_checksum_validation``
+    Determines when checksum validation will be performed on response payloads. Valid values are:
+
+    * ``when_supported`` -- When set, checksum validation is performed on
+      all response payloads of operations modeled with the ``httpChecksum``
+      trait where ``responseAlgorithms`` is modeled, except when no modeled
+      checksum algorithms are supported.
+
+    * ``when_required`` -- When set, checksum validation is not performed
+      on response payloads of operations unless the checksum algorithm is
+      supported and the ``requestValidationModeMember`` member is set to ``ENABLED``.
+
 .. _IAM Roles for Amazon EC2: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
 .. _Using IAM Roles: http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html
 .. _Sourcing Credentials with an External Process: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html

tests/functional/test_s3.py

@@ -16,6 +16,7 @@
 import botocore.stub
 import pytest
 from botocore.config import Config
+from botocore.httpchecksum import DEFAULT_CHECKSUM_ALGORITHM
 from botocore.stub import Stubber
 
 import boto3.session
@@ -86,7 +87,10 @@ def stub_head(self, content_length=4, expected_params=None):
             expected_params=expected_params,
         )
 
-    def stub_create_multipart_upload(self):
+    def stub_create_multipart_upload(
+        self,
+        extra_expected_params=None,
+    ):
         # Add the response and assert params for CreateMultipartUpload
         create_upload_response = {
             "Bucket": self.bucket,
@@ -97,13 +101,17 @@ def stub_create_multipart_upload(self):
             "Bucket": self.bucket,
             "Key": self.key,
         }
+        if extra_expected_params:
+            expected_params.update(extra_expected_params)
         self.stubber.add_response(
             method='create_multipart_upload',
             service_response=create_upload_response,
             expected_params=expected_params,
         )
 
-    def stub_complete_multipart_upload(self, parts):
+    def stub_complete_multipart_upload(
+        self, parts, extra_expected_params=None
+    ):
         complete_upload_response = {
             "Location": "us-west-2",
             "Bucket": self.bucket,
@@ -116,6 +124,8 @@ def stub_complete_multipart_upload(self, parts):
             "MultipartUpload": {"Parts": parts},
             "UploadId": self.upload_id,
         }
+        if extra_expected_params:
+            expected_params.update(extra_expected_params)
 
         self.stubber.add_response(
             method='complete_multipart_upload',
@@ -256,6 +266,7 @@ def stub_put_object(self):
             "Bucket": self.bucket,
             "Key": self.key,
             "Body": botocore.stub.ANY,
+            "ChecksumAlgorithm": DEFAULT_CHECKSUM_ALGORITHM,
         }
         self.stubber.add_response(
             method='put_object',
@@ -267,13 +278,15 @@ def stub_upload_part(self, part_number):
         upload_part_response = {
             'ETag': self.etag,
             'ResponseMetadata': {'HTTPStatusCode': 200},
+            'ChecksumCRC32': f'sum{part_number}==',
         }
         expected_params = {
             "Bucket": self.bucket,
             "Key": self.key,
             "Body": botocore.stub.ANY,
             "PartNumber": part_number,
             "UploadId": self.upload_id,
+            'ChecksumAlgorithm': 'CRC32',
         }
         self.stubber.add_response(
             method='upload_part',
@@ -282,15 +295,25 @@ def stub_upload_part(self, part_number):
         )
 
     def stub_multipart_upload(self, num_parts):
-        self.stub_create_multipart_upload()
+        self.stub_create_multipart_upload(
+            extra_expected_params={
+                "ChecksumAlgorithm": DEFAULT_CHECKSUM_ALGORITHM,
+            }
+        )
 
         # Add the responses for each UploadPartCopy
         parts = []
         for i in range(num_parts):
             # Fill in the parts
             part_number = i + 1
             self.stub_upload_part(part_number=part_number)
-            parts.append({'ETag': self.etag, 'PartNumber': part_number})
+            parts.append(
+                {
+                    'ETag': self.etag,
+                    'PartNumber': part_number,
+                    'ChecksumCRC32': f'sum{part_number}==',
+                }
+            )
 
         self.stub_complete_multipart_upload(parts)