Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Update the logic for checking if a blob exist before uploading #2503

Merged
merged 4 commits into from
Oct 12, 2022
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 33 additions & 37 deletions src/agent/onefuzz/src/uploader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ use anyhow::{Context, Result};
use futures::stream::TryStreamExt;
use reqwest::{Body, Client, Response, StatusCode, Url};
use reqwest_retry::{
send_retry_reqwest_default, RetryCheck, SendRetry, DEFAULT_RETRY_PERIOD, MAX_RETRY_ATTEMPTS,
send_retry_reqwest, send_retry_reqwest_default, RetryCheck, SendRetry, DEFAULT_RETRY_PERIOD,
MAX_RETRY_ATTEMPTS,
};
use serde::Serialize;
use tokio::{fs, io};
Expand Down Expand Up @@ -42,44 +43,39 @@ impl BlobUploader {
url
};

// Check if the file already exists before uploading
if let Ok(head) = self
.client
.head(url.clone())
.send_retry(
|code| match code {
StatusCode::NOT_FOUND => RetryCheck::Fail,
_ => RetryCheck::Retry,
},
DEFAULT_RETRY_PERIOD,
MAX_RETRY_ATTEMPTS,
)
.await
{
if head.status() == StatusCode::OK {
return Ok(head);
}
}

let content_length = format!("{}", file_len);

let resp = send_retry_reqwest_default(|| {
let file = fs::File::from_std(std::fs::File::open(file_path)?);
let reader = io::BufReader::new(file);
let codec = codec::BytesCodec::new();
let file_stream = codec::FramedRead::new(reader, codec)
.map_ok(bytes::BytesMut::freeze)
.into_stream();

let request_builder = self
.client
.put(url.clone())
.header("Content-Length", &content_length)
.header("x-ms-blob-type", "BlockBlob")
.body(Body::wrap_stream(file_stream));

Ok(request_builder)
})
let resp = send_retry_reqwest(
|| {
let file = fs::File::from_std(std::fs::File::open(file_path)?);
let reader = io::BufReader::new(file);
let codec = codec::BytesCodec::new();
let file_stream = codec::FramedRead::new(reader, codec)
.map_ok(bytes::BytesMut::freeze)
.into_stream();

let request_builder = self
.client
.put(url.clone())
// https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob-from-url#request-headers
.header("Content-Length", &content_length)
.header("x-ms-blob-type", "BlockBlob")
// upload only if the the destination blob does not exist
.header("If-None-Match", "*")
.body(Body::wrap_stream(file_stream));

Ok(request_builder)
},
|status| {
if status == StatusCode::PRECONDITION_FAILED {
RetryCheck::Succeed
} else {
RetryCheck::Retry
}
},
DEFAULT_RETRY_PERIOD,
MAX_RETRY_ATTEMPTS,
)
.await
.context("BlobUploader.upload")?;

Expand Down