added CRC32, CRC32C, SHA1, SHA256 and CRC64NVME

Author: HJLebbink

.github/workflows/rust.yml

@@ -36,6 +36,7 @@ jobs:
           export ACCESS_KEY=minioadmin
           export SECRET_KEY=minioadmin
           export ENABLE_HTTPS=1
+          export SERVER_REGION=us-east-1
           export MINIO_SSL_CERT_FILE=./tests/public.crt
           MINIO_TEST_TOKIO_RUNTIME_FLAVOR="multi_thread" cargo test -- --nocapture
   test-current-thread:
@@ -49,6 +50,7 @@ jobs:
           export ACCESS_KEY=minioadmin
           export SECRET_KEY=minioadmin
           export ENABLE_HTTPS=1
+          export SERVER_REGION=us-east-1
           export MINIO_SSL_CERT_FILE=./tests/public.crt
           MINIO_TEST_TOKIO_RUNTIME_FLAVOR="current_thread" cargo test -- --nocapture
 

Cargo.toml

@@ -40,6 +40,8 @@ async-trait = "0.1"
 base64 = "0.22"
 chrono = { version = "0.4", features = ["serde"] }
 crc = "3.3"
+crc32c = "0.6"
+crc32fast = "1.4"
 dashmap = "6.1.0"
 env_logger = "0.11"
 hmac = { version = "0.12", optional = true }
@@ -55,6 +57,7 @@ ring = { version = "0.17", optional = true, default-features = false, features =
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
+sha1 = "0.10"
 sha2 = { version = "0.10", optional = true }
 urlencoding = "2.1"
 xmltree = "0.12"
@@ -95,3 +98,8 @@ name = "load_balancing_with_hooks"
 name = "s3-api"
 path = "benches/s3/api_benchmarks.rs"
 harness = false
+
+[[bench]]
+name = "bench_checksums"
+path = "benches/bench_checksums.rs"
+harness = false

benches/bench_checksums.rs

@@ -0,0 +1,49 @@
+// MinIO Rust Library for Amazon S3 Compatible Cloud Storage
+// Copyright 2025 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use minio::s3::utils::{crc32_checksum, crc32c, crc64nvme_checksum, sha1_hash, sha256_checksum};
+
+fn bench_checksums(c: &mut Criterion) {
+    let sizes = vec![
+        ("1KB", 1024),
+        ("10KB", 10 * 1024),
+        ("100KB", 100 * 1024),
+        ("1MB", 1024 * 1024),
+        ("10MB", 10 * 1024 * 1024),
+    ];
+
+    for (name, size) in sizes {
+        let data = vec![0u8; size];
+
+        let mut group = c.benchmark_group(format!("checksum_{}", name));
+        group.throughput(Throughput::Bytes(size as u64));
+
+        group.bench_function("CRC32", |b| b.iter(|| crc32_checksum(&data)));
+
+        group.bench_function("CRC32C", |b| b.iter(|| crc32c(&data)));
+
+        group.bench_function("CRC64NVME", |b| b.iter(|| crc64nvme_checksum(&data)));
+
+        group.bench_function("SHA1", |b| b.iter(|| sha1_hash(&data)));
+
+        group.bench_function("SHA256", |b| b.iter(|| sha256_checksum(&data)));
+
+        group.finish();
+    }
+}
+
+criterion_group!(benches, bench_checksums);
+criterion_main!(benches);

common/src/example.rs

@@ -52,7 +52,7 @@ pub fn create_bucket_notification_config_example() -> NotificationConfig {
             suffix_filter_rule: Some(SuffixFilterRule {
                 value: String::from("pg"),
             }),
-            queue: String::from("arn:minio:sqs::miniojavatest:webhook"),
+            queue: String::from("arn:minio:sqs:us-east-1:miniojavatest:webhook"),
         }]),
         ..Default::default()
     }

src/s3/builders/append_object.rs

@@ -26,7 +26,9 @@ use crate::s3::response_traits::HasObjectSize;
 use crate::s3::segmented_bytes::SegmentedBytes;
 use crate::s3::sse::Sse;
 use crate::s3::types::{S3Api, S3Request, ToS3Request};
-use crate::s3::utils::{check_bucket_name, check_object_name, check_sse};
+use crate::s3::utils::{
+    ChecksumAlgorithm, check_bucket_name, check_object_name, check_sse, compute_checksum_sb,
+};
 use http::Method;
 use std::sync::Arc;
 use typed_builder::TypedBuilder;
@@ -64,6 +66,14 @@ pub struct AppendObject {
     #[builder(!default)] // force required
     /// value of x-amz-write-offset-bytes
     offset_bytes: u64,
+
+    /// Optional checksum algorithm for data integrity verification during append.
+    ///
+    /// When specified, computes a checksum of the appended data using the selected algorithm
+    /// (CRC32, CRC32C, SHA1, SHA256, or CRC64NVME). The checksum is sent with the append
+    /// operation and verified by the server.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<ChecksumAlgorithm>,
 }
 
 impl S3Api for AppendObject {
@@ -83,6 +93,7 @@ pub type AppendObjectBldr = AppendObjectBuilder<(
     (),
     (Arc<SegmentedBytes>,),
     (u64,),
+    (),
 )>;
 
 impl ToS3Request for AppendObject {
@@ -94,6 +105,21 @@ impl ToS3Request for AppendObject {
         let mut headers: Multimap = self.extra_headers.unwrap_or_default();
         headers.add(X_AMZ_WRITE_OFFSET_BYTES, self.offset_bytes.to_string());
 
+        if let Some(algorithm) = self.checksum_algorithm {
+            let checksum_value = compute_checksum_sb(algorithm, &self.data);
+            headers.add(X_AMZ_CHECKSUM_ALGORITHM, algorithm.as_str().to_string());
+
+            match algorithm {
+                ChecksumAlgorithm::CRC32 => headers.add(X_AMZ_CHECKSUM_CRC32, checksum_value),
+                ChecksumAlgorithm::CRC32C => headers.add(X_AMZ_CHECKSUM_CRC32C, checksum_value),
+                ChecksumAlgorithm::SHA1 => headers.add(X_AMZ_CHECKSUM_SHA1, checksum_value),
+                ChecksumAlgorithm::SHA256 => headers.add(X_AMZ_CHECKSUM_SHA256, checksum_value),
+                ChecksumAlgorithm::CRC64NVME => {
+                    headers.add(X_AMZ_CHECKSUM_CRC64NVME, checksum_value)
+                }
+            }
+        }
+
         Ok(S3Request::builder()
             .client(self.client)
             .method(Method::PUT)
@@ -144,6 +170,13 @@ pub struct AppendObjectContent {
     /// Value of x-amz-write-offset-bytes
     #[builder(default)]
     offset_bytes: u64,
+    /// Optional checksum algorithm for data integrity verification during append.
+    ///
+    /// When specified, computes checksums for appended data using the selected algorithm
+    /// (CRC32, CRC32C, SHA1, SHA256, or CRC64NVME). The checksum is computed for each
+    /// chunk and sent with the append operation.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<ChecksumAlgorithm>,
 }
 
 /// Builder type for [`AppendObjectContent`] that is returned by [`MinioClient::append_object_content`](crate::s3::client::MinioClient::append_object_content).
@@ -162,6 +195,7 @@ pub type AppendObjectContentBldr = AppendObjectContentBuilder<(
     (),
     (),
     (),
+    (),
 )>;
 
 impl AppendObjectContent {
@@ -229,6 +263,7 @@ impl AppendObjectContent {
                 offset_bytes: current_file_size,
                 sse: self.sse,
                 data: Arc::new(seg_bytes),
+                checksum_algorithm: self.checksum_algorithm,
             };
             ao.send().await
         } else if let Some(expected) = object_size.value()
@@ -296,6 +331,7 @@ impl AppendObjectContent {
                 sse: self.sse.clone(),
                 data: Arc::new(part_content),
                 offset_bytes: next_offset_bytes,
+                checksum_algorithm: self.checksum_algorithm,
             };
             let resp: AppendObjectResponse = append_object.send().await?;
             //println!("AppendObjectResponse: object_size={:?}", resp.object_size);

src/s3/builders/copy_object.rs

@@ -23,12 +23,13 @@ use crate::s3::response::{
     CopyObjectInternalResponse, CopyObjectResponse, CreateMultipartUploadResponse,
     StatObjectResponse, UploadPartCopyResponse,
 };
+use crate::s3::response_traits::HasChecksumHeaders;
 use crate::s3::response_traits::HasEtagFromBody;
 use crate::s3::sse::{Sse, SseCustomerKey};
 use crate::s3::types::{Directive, PartInfo, Retention, S3Api, S3Request, ToS3Request};
 use crate::s3::utils::{
-    UtcTime, check_bucket_name, check_object_name, check_sse, check_ssec, to_http_header_value,
-    to_iso8601utc, url_encode,
+    ChecksumAlgorithm, UtcTime, check_bucket_name, check_object_name, check_sse, check_ssec,
+    to_http_header_value, to_iso8601utc, url_encode,
 };
 use async_recursion::async_recursion;
 use http::Method;
@@ -59,6 +60,13 @@ pub struct UploadPartCopy {
     part_number: u16,
     #[builder(default)]
     headers: Multimap,
+    /// Optional checksum algorithm for data integrity verification during part copy.
+    ///
+    /// When specified, the server computes a checksum of the copied part data using
+    /// this algorithm. Use the same algorithm for all parts in a multipart upload.
+    /// Supported algorithms: CRC32, CRC32C, SHA1, SHA256, CRC64NVME.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 impl S3Api for UploadPartCopy {
@@ -78,6 +86,7 @@ pub type UploadPartCopyBldr = UploadPartCopyBuilder<(
     (String,),
     (),
     (),
+    (),
 )>;
 
 impl ToS3Request for UploadPartCopy {
@@ -100,6 +109,10 @@ impl ToS3Request for UploadPartCopy {
         let mut headers: Multimap = self.extra_headers.unwrap_or_default();
         headers.add_multimap(self.headers);
 
+        if let Some(algorithm) = self.checksum_algorithm {
+            headers.add(X_AMZ_CHECKSUM_ALGORITHM, algorithm.as_str().to_string());
+        }
+
         let mut query_params: Multimap = self.extra_query_params.unwrap_or_default();
         {
             query_params.add("partNumber", self.part_number.to_string());
@@ -150,6 +163,8 @@ pub struct CopyObjectInternal {
     metadata_directive: Option<Directive>,
     #[builder(default, setter(into))]
     tagging_directive: Option<Directive>,
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 impl S3Api for CopyObjectInternal {
@@ -175,6 +190,7 @@ pub type CopyObjectInternalBldr = CopyObjectInternalBuilder<(
     (),
     (),
     (),
+    (),
 )>;
 
 impl ToS3Request for CopyObjectInternal {
@@ -261,6 +277,10 @@ impl ToS3Request for CopyObjectInternal {
             if let Some(v) = self.source.ssec {
                 headers.add_multimap(v.copy_headers());
             }
+
+            if let Some(algorithm) = self.checksum_algorithm {
+                headers.add(X_AMZ_CHECKSUM_ALGORITHM, algorithm.as_str().to_string());
+            }
         };
 
         Ok(S3Request::builder()
@@ -310,6 +330,13 @@ pub struct CopyObject {
     metadata_directive: Option<Directive>,
     #[builder(default, setter(into))]
     tagging_directive: Option<Directive>,
+    /// Optional checksum algorithm for data integrity verification during copy.
+    ///
+    /// When specified, the server computes a checksum of the destination object using
+    /// this algorithm during the copy operation. Supported algorithms: CRC32, CRC32C,
+    /// SHA1, SHA256, CRC64NVME. The checksum value is included in response headers for verification.
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<crate::s3::utils::ChecksumAlgorithm>,
 }
 
 /// Builder type for [`CopyObject`] that is returned by [`MinioClient::copy_object`](crate::s3::client::MinioClient::copy_object).
@@ -331,6 +358,7 @@ pub type CopyObjectBldr = CopyObjectBuilder<(
     (),
     (),
     (),
+    (),
 )>;
 
 impl CopyObject {
@@ -434,6 +462,7 @@ impl CopyObject {
                 .source(self.source)
                 .metadata_directive(self.metadata_directive)
                 .tagging_directive(self.tagging_directive)
+                .checksum_algorithm(self.checksum_algorithm)
                 .build()
                 .send()
                 .await?;
@@ -472,6 +501,8 @@ pub struct ComposeObjectInternal {
     legal_hold: bool,
     #[builder(default)]
     sources: Vec<ComposeSource>,
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<ChecksumAlgorithm>,
 }
 
 /// Builder type for [`ComposeObjectInternal`] that is returned by `compose_object_internal` method.
@@ -491,6 +522,7 @@ pub type ComposeObjectInternalBldr = ComposeObjectInternalBuilder<(
     (),
     (),
     (),
+    (),
 )>;
 
 impl ComposeObjectInternal {
@@ -554,6 +586,7 @@ impl ComposeObjectInternal {
                 .extra_query_params(self.extra_query_params.clone())
                 .region(self.region.clone())
                 .extra_headers(Some(headers))
+                .checksum_algorithm(self.checksum_algorithm)
                 .build()
                 .send()
                 .await
@@ -625,11 +658,10 @@ impl ComposeObjectInternal {
                         Err(e) => return (Err(e.into()), upload_id),
                     };
 
-                    parts.push(PartInfo {
-                        number: part_number,
-                        etag,
-                        size,
-                    });
+                    let checksum = self
+                        .checksum_algorithm
+                        .and_then(|alg| resp.get_checksum(alg).map(|v| (alg, v)));
+                    parts.push(PartInfo::new(part_number, etag, size, checksum));
                 } else {
                     while size > 0 {
                         part_number += 1;
@@ -665,11 +697,10 @@ impl ComposeObjectInternal {
                             Err(e) => return (Err(e.into()), upload_id),
                         };
 
-                        parts.push(PartInfo {
-                            number: part_number,
-                            etag,
-                            size,
-                        });
+                        let checksum = self
+                            .checksum_algorithm
+                            .and_then(|alg| resp.get_checksum(alg).map(|v| (alg, v)));
+                        parts.push(PartInfo::new(part_number, etag, size, checksum));
 
                         offset += length;
                         size -= length;
@@ -732,6 +763,8 @@ pub struct ComposeObject {
     legal_hold: bool,
     #[builder(default)]
     sources: Vec<ComposeSource>,
+    #[builder(default, setter(into))]
+    checksum_algorithm: Option<ChecksumAlgorithm>,
 }
 
 /// Builder type for [`ComposeObject`] that is returned by [`MinioClient::compose_object`](crate::s3::client::MinioClient::compose_object).
@@ -751,6 +784,7 @@ pub type ComposeObjectBldr = ComposeObjectBuilder<(
     (),
     (),
     (Vec<ComposeSource>,),
+    (),
 )>;
 
 impl ComposeObject {
@@ -773,6 +807,7 @@ impl ComposeObject {
             .retention(self.retention)
             .legal_hold(self.legal_hold)
             .sources(self.sources)
+            .checksum_algorithm(self.checksum_algorithm)
             .build()
             .send()
             .await;