Skip to content

Commit 2e94a0e

Browse files
authored
Removed unnecessary intermediate Vec<Bytes> allocation in HTTP request body preparation (#192)
- Removed unnecessary intermediate `Vec<Bytes>` allocation in HTTP request body preparation - Directly consume `SegmentedBytes` iterator into stream using `Arc::unwrap_or_clone()` - Implemented zero-cost `BodyIterator` enum to avoid heap allocation and dynamic dispatch - Reduces memory overhead and eliminates Vec growth reallocations during request preparation
1 parent ec0e792 commit 2e94a0e

File tree

1 file changed

+32
-7
lines changed

1 file changed

+32
-7
lines changed

src/s3/client/mod.rs

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,24 @@ pub const MAX_PART_SIZE: u64 = 5_368_709_120; // 5 GiB
119119
/// ensuring the object does not exceed 5 TiB.
120120
pub const MAX_OBJECT_SIZE: u64 = 5_497_558_138_880; // 5 TiB
121121

122+
enum BodyIterator {
123+
Segmented(crate::s3::segmented_bytes::SegmentedBytesIntoIterator),
124+
FromVec(std::vec::IntoIter<Bytes>),
125+
Empty(std::iter::Empty<Bytes>),
126+
}
127+
128+
impl Iterator for BodyIterator {
129+
type Item = Bytes;
130+
131+
fn next(&mut self) -> Option<Self::Item> {
132+
match self {
133+
Self::Segmented(iter) => iter.next(),
134+
Self::FromVec(iter) => iter.next(),
135+
Self::Empty(iter) => iter.next(),
136+
}
137+
}
138+
}
139+
122140
/// Maximum number of parts allowed in a multipart upload.
123141
///
124142
/// Multipart uploads are limited to a total of 10,000 parts. If the object
@@ -540,14 +558,21 @@ impl MinioClient {
540558
}
541559

542560
if (*method == Method::PUT) || (*method == Method::POST) {
543-
//TODO: why-oh-why first collect into a vector and then iterate to a stream?
544-
let bytes_vec: Vec<Bytes> = match body.as_ref() {
545-
Some(v) => v.iter().collect(),
546-
None => Vec::new(),
561+
let iter = match body {
562+
Some(v) => {
563+
// Try to unwrap the Arc if we're the sole owner (zero-cost).
564+
// Otherwise, collect into a Vec to avoid cloning the SegmentedBytes structure.
565+
match Arc::try_unwrap(v) {
566+
Ok(segmented) => BodyIterator::Segmented(segmented.into_iter()),
567+
Err(arc) => {
568+
let vec: Vec<Bytes> = arc.iter().collect();
569+
BodyIterator::FromVec(vec.into_iter())
570+
}
571+
}
572+
}
573+
None => BodyIterator::Empty(std::iter::empty()),
547574
};
548-
let stream = futures_util::stream::iter(
549-
bytes_vec.into_iter().map(|b| -> Result<_, Error> { Ok(b) }),
550-
);
575+
let stream = futures_util::stream::iter(iter.map(|b| -> Result<_, Error> { Ok(b) }));
551576
req = req.body(Body::wrap_stream(stream));
552577
}
553578

0 commit comments

Comments
 (0)