Skip to content

Commit

Permalink
Small file upload optimization
Browse files Browse the repository at this point in the history
If the entire object is uploaded during a single `Write` call, then
skip the multipart upload and just do a single non-buffered upload.
  • Loading branch information
bbockelm committed Nov 30, 2024
1 parent 8d86555 commit 372cbba
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 2 deletions.
2 changes: 1 addition & 1 deletion src/S3Commands.cc
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ bool AmazonRequest::SendS3Request(const std::string_view payload,

AmazonS3Upload::~AmazonS3Upload() {}

bool AmazonS3Upload::SendRequest(const std::string &payload) {
bool AmazonS3Upload::SendRequest(const std::string_view &payload) {
httpVerb = "PUT";
return SendS3Request(payload, payload.size(), true);
}
Expand Down
2 changes: 1 addition & 1 deletion src/S3Commands.hh
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ class AmazonS3Upload final : public AmazonRequest {

virtual ~AmazonS3Upload();

bool SendRequest(const std::string &payload);
bool SendRequest(const std::string_view &payload);

protected:
std::string path;
Expand Down
19 changes: 19 additions & 0 deletions src/S3File.cc
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,25 @@ ssize_t S3File::Write(const void *buffer, off_t offset, size_t size) {
}
std::lock_guard lk(*write_mutex);

// Small object optimization -- if this is the full object, upload
// it immediately.
if (!m_write_offset && m_object_size == size) {
AmazonS3Upload upload(m_ai, m_object, m_log);
m_write_lk.reset();
if (!upload.SendRequest(
std::string_view(static_cast<const char *>(buffer), size))) {
m_log.Log(LogMask::Warning, "Write",
"Failed to create small object");
return -EIO;
} else {
m_write_offset += size;
m_log.Log(LogMask::Debug, "Write",
"Creation of small object succeeded",
std::to_string(size).c_str());
return size;
}
}

if (offset != m_write_offset) {
m_log.Emsg(
"Write",
Expand Down

0 comments on commit 372cbba

Please sign in to comment.