added CRC32, CRC32C, SHA1, SHA256 and CRC64NVME

Author: HJLebbink

Cargo.toml

@@ -40,6 +40,8 @@ async-trait = "0.1"
 base64 = "0.22"
 chrono = { version = "0.4", features = ["serde"] }
 crc = "3.3"
+crc32c = "0.6"
+crc32fast = "1.4"
 dashmap = "6.1.0"
 env_logger = "0.11"
 hmac = { version = "0.12", optional = true }
@@ -55,6 +57,7 @@ ring = { version = "0.17", optional = true, default-features = false, features =
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
+sha1 = "0.10"
 sha2 = { version = "0.10", optional = true }
 urlencoding = "2.1"
 xmltree = "0.12"
@@ -95,3 +98,8 @@ name = "load_balancing_with_hooks"
 name = "s3-api"
 path = "benches/s3/api_benchmarks.rs"
 harness = false
+
+[[bench]]
+name = "bench_checksums"
+path = "benches/bench_checksums.rs"
+harness = false

benches/bench_checksums.rs

@@ -0,0 +1,49 @@
+// MinIO Rust Library for Amazon S3 Compatible Cloud Storage
+// Copyright 2025 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use minio::s3::utils::{crc32_checksum, crc32c, crc64nvme_checksum, sha1_hash, sha256_checksum};
+
+fn bench_checksums(c: &mut Criterion) {
+    let sizes = vec![
+        ("1KB", 1024),
+        ("10KB", 10 * 1024),
+        ("100KB", 100 * 1024),
+        ("1MB", 1024 * 1024),
+        ("10MB", 10 * 1024 * 1024),
+    ];
+
+    for (name, size) in sizes {
+        let data = vec![0u8; size];
+
+        let mut group = c.benchmark_group(format!("checksum_{}", name));
+        group.throughput(Throughput::Bytes(size as u64));
+
+        group.bench_function("CRC32", |b| b.iter(|| crc32_checksum(&data)));
+
+        group.bench_function("CRC32C", |b| b.iter(|| crc32c(&data)));
+
+        group.bench_function("CRC64NVME", |b| b.iter(|| crc64nvme_checksum(&data)));
+
+        group.bench_function("SHA1", |b| b.iter(|| sha1_hash(&data)));
+
+        group.bench_function("SHA256", |b| b.iter(|| sha256_checksum(&data)));
+
+        group.finish();
+    }
+}
+
+criterion_group!(benches, bench_checksums);
+criterion_main!(benches);

src/s3/builders/append_object.rs

@@ -64,6 +64,14 @@ pub struct AppendObject {
     #[builder(!default)] // force required
     /// value of x-amz-write-offset-bytes
     offset_bytes: u64,
+
+    /// Optional checksum algorithm for data integrity verification during append.
+    ///
+    /// When specified, computes a checksum of the appended data using the selected algorithm
+    /// (CRC32, CRC32C, SHA1, SHA256, or CRC64NVME). The checksum is sent with the append
+    /// operation and verified by the server.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 impl S3Api for AppendObject {
@@ -83,6 +91,7 @@ pub type AppendObjectBldr = AppendObjectBuilder<(
     (),
     (Arc<SegmentedBytes>,),
     (u64,),
+    (),
 )>;
 
 impl ToS3Request for AppendObject {
@@ -94,6 +103,24 @@ impl ToS3Request for AppendObject {
         let mut headers: Multimap = self.extra_headers.unwrap_or_default();
         headers.add(X_AMZ_WRITE_OFFSET_BYTES, self.offset_bytes.to_string());
 
+        if let Some(algorithm) = self.checksum_algorithm {
+            use crate::s3::header_constants::*;
+            use crate::s3::utils::{ChecksumAlgorithm, compute_checksum_sb};
+
+            let checksum_value = compute_checksum_sb(algorithm, &self.data);
+            headers.add(X_AMZ_CHECKSUM_ALGORITHM, algorithm.as_str().to_string());
+
+            match algorithm {
+                ChecksumAlgorithm::CRC32 => headers.add(X_AMZ_CHECKSUM_CRC32, checksum_value),
+                ChecksumAlgorithm::CRC32C => headers.add(X_AMZ_CHECKSUM_CRC32C, checksum_value),
+                ChecksumAlgorithm::SHA1 => headers.add(X_AMZ_CHECKSUM_SHA1, checksum_value),
+                ChecksumAlgorithm::SHA256 => headers.add(X_AMZ_CHECKSUM_SHA256, checksum_value),
+                ChecksumAlgorithm::CRC64NVME => {
+                    headers.add(X_AMZ_CHECKSUM_CRC64NVME, checksum_value)
+                }
+            }
+        }
+
         Ok(S3Request::builder()
             .client(self.client)
             .method(Method::PUT)
@@ -144,6 +171,13 @@ pub struct AppendObjectContent {
     /// Value of x-amz-write-offset-bytes
     #[builder(default)]
     offset_bytes: u64,
+    /// Optional checksum algorithm for data integrity verification during append.
+    ///
+    /// When specified, computes checksums for appended data using the selected algorithm
+    /// (CRC32, CRC32C, SHA1, SHA256, or CRC64NVME). The checksum is computed for each
+    /// chunk and sent with the append operation.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 /// Builder type for [`AppendObjectContent`] that is returned by [`MinioClient::append_object_content`](crate::s3::client::MinioClient::append_object_content).
@@ -162,6 +196,7 @@ pub type AppendObjectContentBldr = AppendObjectContentBuilder<(
     (),
     (),
     (),
+    (),
 )>;
 
 impl AppendObjectContent {
@@ -229,6 +264,7 @@ impl AppendObjectContent {
                 offset_bytes: current_file_size,
                 sse: self.sse,
                 data: Arc::new(seg_bytes),
+                checksum_algorithm: self.checksum_algorithm,
             };
             ao.send().await
         } else if let Some(expected) = object_size.value()
@@ -296,6 +332,7 @@ impl AppendObjectContent {
                 sse: self.sse.clone(),
                 data: Arc::new(part_content),
                 offset_bytes: next_offset_bytes,
+                checksum_algorithm: self.checksum_algorithm,
             };
             let resp: AppendObjectResponse = append_object.send().await?;
             //println!("AppendObjectResponse: object_size={:?}", resp.object_size);

src/s3/builders/copy_object.rs

@@ -23,12 +23,12 @@ use crate::s3::response::{
     CopyObjectInternalResponse, CopyObjectResponse, CreateMultipartUploadResponse,
     StatObjectResponse, UploadPartCopyResponse,
 };
-use crate::s3::response_traits::HasEtagFromBody;
+use crate::s3::response_traits::{HasChecksumHeaders, HasEtagFromBody};
 use crate::s3::sse::{Sse, SseCustomerKey};
 use crate::s3::types::{Directive, PartInfo, Retention, S3Api, S3Request, ToS3Request};
 use crate::s3::utils::{
-    UtcTime, check_bucket_name, check_object_name, check_sse, check_ssec, to_http_header_value,
-    to_iso8601utc, url_encode,
+    ChecksumAlgorithm, UtcTime, check_bucket_name, check_object_name, check_sse, check_ssec,
+    to_http_header_value, to_iso8601utc, url_encode,
 };
 use async_recursion::async_recursion;
 use http::Method;
@@ -59,6 +59,13 @@ pub struct UploadPartCopy {
     part_number: u16,
     #[builder(default)]
     headers: Multimap,
+    /// Optional checksum algorithm for data integrity verification during part copy.
+    ///
+    /// When specified, the server computes a checksum of the copied part data using
+    /// this algorithm. Use the same algorithm for all parts in a multipart upload.
+    /// Supported algorithms: CRC32, CRC32C, SHA1, SHA256, CRC64NVME.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 impl S3Api for UploadPartCopy {
@@ -78,6 +85,7 @@ pub type UploadPartCopyBldr = UploadPartCopyBuilder<(
     (String,),
     (),
     (),
+    (),
 )>;
 
 impl ToS3Request for UploadPartCopy {
@@ -100,6 +108,10 @@ impl ToS3Request for UploadPartCopy {
         let mut headers: Multimap = self.extra_headers.unwrap_or_default();
         headers.add_multimap(self.headers);
 
+        if let Some(algorithm) = self.checksum_algorithm {
+            headers.add(X_AMZ_CHECKSUM_ALGORITHM, algorithm.as_str().to_string());
+        }
+
         let mut query_params: Multimap = self.extra_query_params.unwrap_or_default();
         {
             query_params.add("partNumber", self.part_number.to_string());
@@ -150,6 +162,8 @@ pub struct CopyObjectInternal {
     metadata_directive: Option<Directive>,
     #[builder(default, setter(into))]
     tagging_directive: Option<Directive>,
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 impl S3Api for CopyObjectInternal {
@@ -175,6 +189,7 @@ pub type CopyObjectInternalBldr = CopyObjectInternalBuilder<(
     (),
     (),
     (),
+    (),
 )>;
 
 impl ToS3Request for CopyObjectInternal {
@@ -261,6 +276,10 @@ impl ToS3Request for CopyObjectInternal {
             if let Some(v) = self.source.ssec {
                 headers.add_multimap(v.copy_headers());
             }
+
+            if let Some(algorithm) = self.checksum_algorithm {
+                headers.add(X_AMZ_CHECKSUM_ALGORITHM, algorithm.as_str().to_string());
+            }
         };
 
         Ok(S3Request::builder()
@@ -310,6 +329,13 @@ pub struct CopyObject {
     metadata_directive: Option<Directive>,
     #[builder(default, setter(into))]
     tagging_directive: Option<Directive>,
+    /// Optional checksum algorithm for data integrity verification during copy.
+    ///
+    /// When specified, the server computes a checksum of the destination object using
+    /// this algorithm during the copy operation. Supported algorithms: CRC32, CRC32C,
+    /// SHA1, SHA256, CRC64NVME. The checksum value is included in response headers for verification.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 /// Builder type for [`CopyObject`] that is returned by [`MinioClient::copy_object`](crate::s3::client::MinioClient::copy_object).
@@ -331,6 +357,7 @@ pub type CopyObjectBldr = CopyObjectBuilder<(
     (),
     (),
     (),
+    (),
 )>;
 
 impl CopyObject {
@@ -434,6 +461,7 @@ impl CopyObject {
                 .source(self.source)
                 .metadata_directive(self.metadata_directive)
                 .tagging_directive(self.tagging_directive)
+                .checksum_algorithm(self.checksum_algorithm)
                 .build()
                 .send()
                 .await?;
@@ -629,6 +657,11 @@ impl ComposeObjectInternal {
                         number: part_number,
                         etag,
                         size,
+                        checksum_crc32: resp.get_checksum(ChecksumAlgorithm::CRC32),
+                        checksum_crc32c: resp.get_checksum(ChecksumAlgorithm::CRC32C),
+                        checksum_sha1: resp.get_checksum(ChecksumAlgorithm::SHA1),
+                        checksum_sha256: resp.get_checksum(ChecksumAlgorithm::SHA256),
+                        checksum_crc64nvme: resp.get_checksum(ChecksumAlgorithm::CRC64NVME),
                     });
                 } else {
                     while size > 0 {
@@ -669,6 +702,11 @@ impl ComposeObjectInternal {
                             number: part_number,
                             etag,
                             size,
+                            checksum_crc32: resp.get_checksum(ChecksumAlgorithm::CRC32),
+                            checksum_crc32c: resp.get_checksum(ChecksumAlgorithm::CRC32C),
+                            checksum_sha1: resp.get_checksum(ChecksumAlgorithm::SHA1),
+                            checksum_sha256: resp.get_checksum(ChecksumAlgorithm::SHA256),
+                            checksum_crc64nvme: resp.get_checksum(ChecksumAlgorithm::CRC64NVME),
                         });
 
                         offset += length;