From ba232df0ca8fc93613a59f8b105f85dd9fd1866a Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Fri, 7 Jan 2022 17:11:18 -0800 Subject: [PATCH 1/8] Inital changes needed for backoff 0.4 to work --- src/agent/onefuzz/src/az_copy.rs | 3 ++- src/agent/reqwest-retry/src/lib.rs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/agent/onefuzz/src/az_copy.rs b/src/agent/onefuzz/src/az_copy.rs index e9099399d4..51f348959c 100644 --- a/src/agent/onefuzz/src/az_copy.rs +++ b/src/agent/onefuzz/src/az_copy.rs @@ -145,6 +145,7 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R failure_count + 1 ) }); + let option_retry_interval: Option = Some(RETRY_INTERVAL); match result { Ok(()) => Ok(()), Err(err) => { @@ -154,7 +155,7 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R if failure_count >= RETRY_COUNT { Err(backoff::Error::Permanent(err)) } else { - Err(backoff::Error::Transient(err)) + Err(backoff::Error::Transient{err: err, retry_after: option_retry_interval}) } } } diff --git a/src/agent/reqwest-retry/src/lib.rs b/src/agent/reqwest-retry/src/lib.rs index bed5f77980..6d67e622d0 100644 --- a/src/agent/reqwest-retry/src/lib.rs +++ b/src/agent/reqwest-retry/src/lib.rs @@ -56,13 +56,13 @@ where .send() .await .with_context(|| format!("request attempt {} failed", attempt_count + 1)); - + let option_default_retry_period: Option = Some(DEFAULT_RETRY_PERIOD); match result { Err(x) => { if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(x))) } else { - Err(backoff::Error::Transient(Err(x))) + Err(backoff::Error::Transient{err: Err(x), retry_after: option_default_retry_period}) } } Ok(x) => { @@ -93,7 +93,7 @@ where if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(as_err))) } else { - Err(backoff::Error::Transient(Err(as_err))) + Err(backoff::Error::Transient{err: Err(as_err), retry_after: option_default_retry_period}) } } } From ce9367fd1e5815f1bbbb5fee73013054bd56e1a7 Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Thu, 13 Jan 2022 12:02:40 -0800 Subject: [PATCH 2/8] Update backoff versions, fix BackoffError:Transient fields, other uses --- src/agent/onefuzz-agent/Cargo.toml | 2 +- src/agent/onefuzz-agent/src/local/common.rs | 6 ++---- src/agent/onefuzz/Cargo.toml | 2 +- src/agent/reqwest-retry/Cargo.toml | 2 +- src/agent/reqwest-retry/src/lib.rs | 2 +- src/agent/storage-queue/Cargo.toml | 2 +- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/agent/onefuzz-agent/Cargo.toml b/src/agent/onefuzz-agent/Cargo.toml index 9cfbfb7892..cbbaa04312 100644 --- a/src/agent/onefuzz-agent/Cargo.toml +++ b/src/agent/onefuzz-agent/Cargo.toml @@ -14,7 +14,7 @@ anyhow = "1.0" arraydeque = "0.4" async-trait = "0.1" atexit = { path = "../atexit" } -backoff = { version = "0.3", features = ["tokio"] } +backoff = { version = "0.4", features = ["tokio"] } clap = "2.34" coverage = { path = "../coverage" } crossterm = "0.22" diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 2bcd85ca20..caccc29856 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -299,10 +299,8 @@ pub async fn wait_for_dir(path: impl AsRef) -> Result<()> { if path.as_ref().exists() { Ok(()) } else { - Err(BackoffError::Transient(anyhow::anyhow!( - "path '{:?}' does not exist", - path.as_ref() - ))) + let max_retry_period: Option = Some(WAIT_FOR_MAX_WAIT); + Err(BackoffError::Transient{err: anyhow::anyhow!("path '{:?}' does not exist", path.as_ref()), retry_after: max_retry_period}) } }; retry( diff --git a/src/agent/onefuzz/Cargo.toml b/src/agent/onefuzz/Cargo.toml index 074551b928..68564e807d 100644 --- a/src/agent/onefuzz/Cargo.toml +++ b/src/agent/onefuzz/Cargo.toml @@ -41,7 +41,7 @@ process_control = "3.0" reqwest-retry = { path = "../reqwest-retry"} onefuzz-telemetry = { path = "../onefuzz-telemetry"} stacktrace-parser = { path = "../stacktrace-parser" } -backoff = { version = "0.3", features = ["tokio"] } +backoff = { version = "0.4", features = ["tokio"] } [target.'cfg(target_family = "windows")'.dependencies] winreg = "0.10" diff --git a/src/agent/reqwest-retry/Cargo.toml b/src/agent/reqwest-retry/Cargo.toml index 35c6805a10..f9c314e0d0 100644 --- a/src/agent/reqwest-retry/Cargo.toml +++ b/src/agent/reqwest-retry/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT" [dependencies] anyhow = "1.0" async-trait = "0.1" -backoff = { version = "0.3", features = ["tokio"] } +backoff = { version = "0.4", features = ["tokio"] } log = "0.4" onefuzz-telemetry = { path = "../onefuzz-telemetry" } reqwest = { version = "0.11", features = ["json", "stream", "rustls-tls"], default-features=false } diff --git a/src/agent/reqwest-retry/src/lib.rs b/src/agent/reqwest-retry/src/lib.rs index 6d67e622d0..a3d147b627 100644 --- a/src/agent/reqwest-retry/src/lib.rs +++ b/src/agent/reqwest-retry/src/lib.rs @@ -80,7 +80,7 @@ where }) { // the is_success check earlier should have taken care of this already. Ok(x) => Ok(x), - Err(as_err) => Err(backoff::Error::Permanent(Err(as_err))), + Err(as_err) => Err(backoff::Error::Permanent(Err(as_err))), } } RetryCheck::Retry => { diff --git a/src/agent/storage-queue/Cargo.toml b/src/agent/storage-queue/Cargo.toml index 32fd653f3e..4a856c33fb 100644 --- a/src/agent/storage-queue/Cargo.toml +++ b/src/agent/storage-queue/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT" [dependencies] anyhow = "1.0" async-trait = "0.1" -backoff = { version = "0.3", features = ["tokio"] } +backoff = { version = "0.4", features = ["tokio"] } base64 = "0.13" bytes = { version = "1.1", features = ["serde"] } derivative = "2.2" From a8f62655dde491feede4c61dc32ce5d56be07ea0 Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Thu, 13 Jan 2022 12:09:02 -0800 Subject: [PATCH 3/8] Format --- src/agent/onefuzz-agent/src/local/common.rs | 5 ++++- src/agent/onefuzz/src/az_copy.rs | 5 ++++- src/agent/reqwest-retry/src/lib.rs | 12 +++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index caccc29856..cb925a848c 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -300,7 +300,10 @@ pub async fn wait_for_dir(path: impl AsRef) -> Result<()> { Ok(()) } else { let max_retry_period: Option = Some(WAIT_FOR_MAX_WAIT); - Err(BackoffError::Transient{err: anyhow::anyhow!("path '{:?}' does not exist", path.as_ref()), retry_after: max_retry_period}) + Err(BackoffError::Transient { + err: anyhow::anyhow!("path '{:?}' does not exist", path.as_ref()), + retry_after: max_retry_period, + }) } }; retry( diff --git a/src/agent/onefuzz/src/az_copy.rs b/src/agent/onefuzz/src/az_copy.rs index 51f348959c..f151faaf0b 100644 --- a/src/agent/onefuzz/src/az_copy.rs +++ b/src/agent/onefuzz/src/az_copy.rs @@ -155,7 +155,10 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R if failure_count >= RETRY_COUNT { Err(backoff::Error::Permanent(err)) } else { - Err(backoff::Error::Transient{err: err, retry_after: option_retry_interval}) + Err(backoff::Error::Transient { + err: err, + retry_after: option_retry_interval, + }) } } } diff --git a/src/agent/reqwest-retry/src/lib.rs b/src/agent/reqwest-retry/src/lib.rs index a3d147b627..4e0649f70c 100644 --- a/src/agent/reqwest-retry/src/lib.rs +++ b/src/agent/reqwest-retry/src/lib.rs @@ -62,7 +62,10 @@ where if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(x))) } else { - Err(backoff::Error::Transient{err: Err(x), retry_after: option_default_retry_period}) + Err(backoff::Error::Transient { + err: Err(x), + retry_after: option_default_retry_period, + }) } } Ok(x) => { @@ -80,7 +83,7 @@ where }) { // the is_success check earlier should have taken care of this already. Ok(x) => Ok(x), - Err(as_err) => Err(backoff::Error::Permanent(Err(as_err))), + Err(as_err) => Err(backoff::Error::Permanent(Err(as_err))), } } RetryCheck::Retry => { @@ -93,7 +96,10 @@ where if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(as_err))) } else { - Err(backoff::Error::Transient{err: Err(as_err), retry_after: option_default_retry_period}) + Err(backoff::Error::Transient { + err: Err(as_err), + retry_after: option_default_retry_period, + }) } } } From d14097f64b1a9f22ffc9f72d6bc5ee5743fd69c4 Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Tue, 18 Jan 2022 15:06:13 -0800 Subject: [PATCH 4/8] Removed redundant field name --- src/agent/onefuzz/src/az_copy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/onefuzz/src/az_copy.rs b/src/agent/onefuzz/src/az_copy.rs index f151faaf0b..6cd30a490d 100644 --- a/src/agent/onefuzz/src/az_copy.rs +++ b/src/agent/onefuzz/src/az_copy.rs @@ -156,7 +156,7 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R Err(backoff::Error::Permanent(err)) } else { Err(backoff::Error::Transient { - err: err, + err, retry_after: option_retry_interval, }) } From d59b5985edf57e9074af1f66443aec50931dfe02 Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Tue, 8 Mar 2022 14:27:37 -0800 Subject: [PATCH 5/8] Improved backoff update changes --- src/agent/onefuzz-agent/src/local/common.rs | 4 +--- src/agent/onefuzz/src/az_copy.rs | 6 +----- src/agent/reqwest-retry/src/lib.rs | 11 ++--------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index cb925a848c..52ff4ff3d7 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -299,10 +299,8 @@ pub async fn wait_for_dir(path: impl AsRef) -> Result<()> { if path.as_ref().exists() { Ok(()) } else { - let max_retry_period: Option = Some(WAIT_FOR_MAX_WAIT); Err(BackoffError::Transient { - err: anyhow::anyhow!("path '{:?}' does not exist", path.as_ref()), - retry_after: max_retry_period, + err: anyhow::anyhow!("path '{}' does not exist", path.as_ref().display()), }) } }; diff --git a/src/agent/onefuzz/src/az_copy.rs b/src/agent/onefuzz/src/az_copy.rs index 6cd30a490d..3ea45d41c7 100644 --- a/src/agent/onefuzz/src/az_copy.rs +++ b/src/agent/onefuzz/src/az_copy.rs @@ -145,7 +145,6 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R failure_count + 1 ) }); - let option_retry_interval: Option = Some(RETRY_INTERVAL); match result { Ok(()) => Ok(()), Err(err) => { @@ -155,10 +154,7 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R if failure_count >= RETRY_COUNT { Err(backoff::Error::Permanent(err)) } else { - Err(backoff::Error::Transient { - err, - retry_after: option_retry_interval, - }) + Err(backoff::Error::transient { err }) } } } diff --git a/src/agent/reqwest-retry/src/lib.rs b/src/agent/reqwest-retry/src/lib.rs index 4e0649f70c..35a86b9506 100644 --- a/src/agent/reqwest-retry/src/lib.rs +++ b/src/agent/reqwest-retry/src/lib.rs @@ -56,16 +56,12 @@ where .send() .await .with_context(|| format!("request attempt {} failed", attempt_count + 1)); - let option_default_retry_period: Option = Some(DEFAULT_RETRY_PERIOD); match result { Err(x) => { if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(x))) } else { - Err(backoff::Error::Transient { - err: Err(x), - retry_after: option_default_retry_period, - }) + Err(backoff::Error::transient { err: Err(x) }) } } Ok(x) => { @@ -96,10 +92,7 @@ where if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(as_err))) } else { - Err(backoff::Error::Transient { - err: Err(as_err), - retry_after: option_default_retry_period, - }) + Err(backoff::Error::transient { err: Err(as_err) }) } } } From 42198115389b0e7be2de1752cd1d182a229b3b4e Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Tue, 8 Mar 2022 14:31:34 -0800 Subject: [PATCH 6/8] Update backoff update --- src/agent/onefuzz-agent/src/local/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 52ff4ff3d7..a01bf596dc 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -299,7 +299,7 @@ pub async fn wait_for_dir(path: impl AsRef) -> Result<()> { if path.as_ref().exists() { Ok(()) } else { - Err(BackoffError::Transient { + Err(BackoffError::transient { err: anyhow::anyhow!("path '{}' does not exist", path.as_ref().display()), }) } From feb1aba9e0b7cef1a9a0ccd0cd7011b00e028c48 Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Tue, 8 Mar 2022 14:39:16 -0800 Subject: [PATCH 7/8] Revert --- src/agent/onefuzz-agent/src/local/common.rs | 4 +++- src/agent/onefuzz/src/az_copy.rs | 6 +++++- src/agent/reqwest-retry/src/lib.rs | 11 +++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index a01bf596dc..90d632721a 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -299,8 +299,10 @@ pub async fn wait_for_dir(path: impl AsRef) -> Result<()> { if path.as_ref().exists() { Ok(()) } else { - Err(BackoffError::transient { + let max_retry_period: Option = Some(WAIT_FOR_MAX_WAIT); + Err(BackoffError::Transient { err: anyhow::anyhow!("path '{}' does not exist", path.as_ref().display()), + retry_after: max_retry_period, }) } }; diff --git a/src/agent/onefuzz/src/az_copy.rs b/src/agent/onefuzz/src/az_copy.rs index 3ea45d41c7..6cd30a490d 100644 --- a/src/agent/onefuzz/src/az_copy.rs +++ b/src/agent/onefuzz/src/az_copy.rs @@ -145,6 +145,7 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R failure_count + 1 ) }); + let option_retry_interval: Option = Some(RETRY_INTERVAL); match result { Ok(()) => Ok(()), Err(err) => { @@ -154,7 +155,10 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R if failure_count >= RETRY_COUNT { Err(backoff::Error::Permanent(err)) } else { - Err(backoff::Error::transient { err }) + Err(backoff::Error::Transient { + err, + retry_after: option_retry_interval, + }) } } } diff --git a/src/agent/reqwest-retry/src/lib.rs b/src/agent/reqwest-retry/src/lib.rs index 35a86b9506..4e0649f70c 100644 --- a/src/agent/reqwest-retry/src/lib.rs +++ b/src/agent/reqwest-retry/src/lib.rs @@ -56,12 +56,16 @@ where .send() .await .with_context(|| format!("request attempt {} failed", attempt_count + 1)); + let option_default_retry_period: Option = Some(DEFAULT_RETRY_PERIOD); match result { Err(x) => { if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(x))) } else { - Err(backoff::Error::transient { err: Err(x) }) + Err(backoff::Error::Transient { + err: Err(x), + retry_after: option_default_retry_period, + }) } } Ok(x) => { @@ -92,7 +96,10 @@ where if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(as_err))) } else { - Err(backoff::Error::transient { err: Err(as_err) }) + Err(backoff::Error::Transient { + err: Err(as_err), + retry_after: option_default_retry_period, + }) } } } From fd3ed49ce3959cd389a8519b8075670cf8b18a23 Mon Sep 17 00:00:00 2001 From: Hayley Call Date: Tue, 8 Mar 2022 15:09:33 -0800 Subject: [PATCH 8/8] Changed to using Error::transient function --- src/agent/onefuzz-agent/src/local/common.rs | 9 ++++----- src/agent/onefuzz/src/az_copy.rs | 6 +----- src/agent/reqwest-retry/src/lib.rs | 11 ++--------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 90d632721a..b1a25ad63d 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -299,11 +299,10 @@ pub async fn wait_for_dir(path: impl AsRef) -> Result<()> { if path.as_ref().exists() { Ok(()) } else { - let max_retry_period: Option = Some(WAIT_FOR_MAX_WAIT); - Err(BackoffError::Transient { - err: anyhow::anyhow!("path '{}' does not exist", path.as_ref().display()), - retry_after: max_retry_period, - }) + Err(BackoffError::transient(anyhow::anyhow!( + "path '{}' does not exist", + path.as_ref().display() + ))) } }; retry( diff --git a/src/agent/onefuzz/src/az_copy.rs b/src/agent/onefuzz/src/az_copy.rs index 6cd30a490d..b4b337dbf4 100644 --- a/src/agent/onefuzz/src/az_copy.rs +++ b/src/agent/onefuzz/src/az_copy.rs @@ -145,7 +145,6 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R failure_count + 1 ) }); - let option_retry_interval: Option = Some(RETRY_INTERVAL); match result { Ok(()) => Ok(()), Err(err) => { @@ -155,10 +154,7 @@ async fn retry_az_impl(mode: Mode, src: &OsStr, dst: &OsStr, args: &[&str]) -> R if failure_count >= RETRY_COUNT { Err(backoff::Error::Permanent(err)) } else { - Err(backoff::Error::Transient { - err, - retry_after: option_retry_interval, - }) + Err(backoff::Error::transient(err)) } } } diff --git a/src/agent/reqwest-retry/src/lib.rs b/src/agent/reqwest-retry/src/lib.rs index 4e0649f70c..216681b3b6 100644 --- a/src/agent/reqwest-retry/src/lib.rs +++ b/src/agent/reqwest-retry/src/lib.rs @@ -56,16 +56,12 @@ where .send() .await .with_context(|| format!("request attempt {} failed", attempt_count + 1)); - let option_default_retry_period: Option = Some(DEFAULT_RETRY_PERIOD); match result { Err(x) => { if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(x))) } else { - Err(backoff::Error::Transient { - err: Err(x), - retry_after: option_default_retry_period, - }) + Err(backoff::Error::transient(Err(x))) } } Ok(x) => { @@ -96,10 +92,7 @@ where if attempt_count >= max_retry { Err(backoff::Error::Permanent(Err(as_err))) } else { - Err(backoff::Error::Transient { - err: Err(as_err), - retry_after: option_default_retry_period, - }) + Err(backoff::Error::transient(Err(as_err))) } } }