feat: send entire object checksum in the final api call of resumable upload (#1654)

feat: send entire object checksum in the final api call of resumable
upload

fixes b/461994245

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: chandra-siri

google/cloud/storage/_media/_upload.py

@@ -688,6 +688,13 @@ def _prepare_request(self):
             _CONTENT_TYPE_HEADER: self._content_type,
             _helpers.CONTENT_RANGE_HEADER: content_range,
         }
+        if (start_byte + len(payload) == self._total_bytes) and (
+            self._checksum_object is not None
+        ):
+            local_checksum = _helpers.prepare_checksum_digest(
+                self._checksum_object.digest()
+            )
+            headers["x-goog-hash"] = f"{self._checksum_type}={local_checksum}"
         return _PUT, self.resumable_url, payload, headers
 
     def _update_checksum(self, start_byte, payload):

tests/resumable_media/system/requests/test_upload.py

@@ -27,7 +27,6 @@
 import google.cloud.storage._media.requests as resumable_requests
 from google.cloud.storage._media import _helpers
 from .. import utils
-from google.cloud.storage._media import _upload
 from google.cloud.storage.exceptions import InvalidResponse
 from google.cloud.storage.exceptions import DataCorruption
 
@@ -372,29 +371,6 @@ def test_resumable_upload_with_headers(
     _resumable_upload_helper(authorized_transport, img_stream, cleanup, headers=headers)
 
 
-@pytest.mark.parametrize("checksum", ["md5", "crc32c"])
-def test_resumable_upload_with_bad_checksum(
-    authorized_transport, img_stream, bucket, cleanup, checksum
-):
-    fake_checksum_object = _helpers._get_checksum_object(checksum)
-    fake_checksum_object.update(b"bad data")
-    fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
-        fake_checksum_object.digest()
-    )
-    with mock.patch.object(
-        _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest
-    ):
-        with pytest.raises(DataCorruption) as exc_info:
-            _resumable_upload_helper(
-                authorized_transport, img_stream, cleanup, checksum=checksum
-            )
-    expected_checksums = {"md5": "1bsd83IYNug8hd+V1ING3Q==", "crc32c": "YQGPxA=="}
-    expected_message = _upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
-        checksum.upper(), fake_prepared_checksum_digest, expected_checksums[checksum]
-    )
-    assert exc_info.value.args[0] == expected_message
-
-
 def test_resumable_upload_bad_chunk_size(authorized_transport, img_stream):
     blob_name = os.path.basename(img_stream.name)
     # Create the actual upload object.

tests/unit/test_blob.py

@@ -3049,15 +3049,22 @@ def test__initiate_resumable_upload_with_client_custom_headers(self):
         self._initiate_resumable_helper(client=client)
 
     def _make_resumable_transport(
-        self, headers1, headers2, headers3, total_bytes, data_corruption=False
+        self,
+        headers1,
+        headers2,
+        headers3,
+        total_bytes,
+        data_corruption=False,
+        md5_checksum_value=None,
+        crc32c_checksum_value=None,
     ):
         fake_transport = mock.Mock(spec=["request"])
 
         fake_response1 = self._mock_requests_response(http.client.OK, headers1)
         fake_response2 = self._mock_requests_response(
             http.client.PERMANENT_REDIRECT, headers2
         )
-        json_body = f'{{"size": "{total_bytes:d}"}}'
+        json_body = json.dumps({"size": str(total_bytes), "md5Hash": md5_checksum_value, "crc32c": crc32c_checksum_value})
         if data_corruption:
             fake_response3 = DataCorruption(None)
         else:
@@ -3151,6 +3158,9 @@ def _do_resumable_upload_call2(
         if_metageneration_match=None,
         if_metageneration_not_match=None,
         timeout=None,
+        checksum=None,
+        crc32c_checksum_value=None,
+        md5_checksum_value=None,
     ):
         # Third mock transport.request() does sends last chunk.
         content_range = f"bytes {blob.chunk_size:d}-{total_bytes - 1:d}/{total_bytes:d}"
@@ -3161,6 +3171,11 @@ def _do_resumable_upload_call2(
             "content-type": content_type,
             "content-range": content_range,
         }
+        if checksum == "crc32c":
+            expected_headers["x-goog-hash"] = f"crc32c={crc32c_checksum_value}"
+        elif checksum == "md5":
+            expected_headers["x-goog-hash"] = f"md5={md5_checksum_value}"
+
         payload = data[blob.chunk_size :]
         return mock.call(
             "PUT",
@@ -3181,12 +3196,17 @@ def _do_resumable_helper(
         timeout=None,
         data_corruption=False,
         retry=None,
+        checksum=None,  # None is also a valid value, when user decides to disable checksum validation.
     ):
         CHUNK_SIZE = 256 * 1024
         USER_AGENT = "testing 1.2.3"
         content_type = "text/html"
         # Data to be uploaded.
         data = b"<html>" + (b"A" * CHUNK_SIZE) + b"</html>"
+
+        # Data calcuated offline and entered here. (Unit test best practice).
+        crc32c_checksum_value = "mQ30hg=="
+        md5_checksum_value = "wajHeg1f2Q2u9afI6fjPOw=="
         total_bytes = len(data)
         if use_size:
             size = total_bytes
@@ -3213,6 +3233,8 @@ def _do_resumable_helper(
                 headers3,
                 total_bytes,
                 data_corruption=data_corruption,
+                md5_checksum_value=md5_checksum_value,
+                crc32c_checksum_value=crc32c_checksum_value,
             )
 
         # Create some mock arguments and call the method under test.
@@ -3247,7 +3269,7 @@ def _do_resumable_helper(
                 if_generation_not_match,
                 if_metageneration_match,
                 if_metageneration_not_match,
-                checksum=None,
+                checksum=checksum,
                 retry=retry,
                 **timeout_kwarg,
             )
@@ -3296,6 +3318,9 @@ def _do_resumable_helper(
                 if_metageneration_match=if_metageneration_match,
                 if_metageneration_not_match=if_metageneration_not_match,
                 timeout=expected_timeout,
+                checksum=checksum,
+                crc32c_checksum_value=crc32c_checksum_value,
+                md5_checksum_value=md5_checksum_value,
             )
         self.assertEqual(transport.request.mock_calls, [call0, call1, call2])
 
@@ -3308,6 +3333,12 @@ def test__do_resumable_upload_no_size(self):
     def test__do_resumable_upload_with_size(self):
         self._do_resumable_helper(use_size=True)
 
+    def test__do_resumable_upload_with_size_with_crc32c_checksum(self):
+        self._do_resumable_helper(use_size=True, checksum="crc32c")
+
+    def test__do_resumable_upload_with_size_with_md5_checksum(self):
+        self._do_resumable_helper(use_size=True, checksum="md5")
+
     def test__do_resumable_upload_with_retry(self):
         self._do_resumable_helper(retry=DEFAULT_RETRY)