From 281538cc4e3c9f223f3b4d01f324103a1be79efc Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Mon, 11 Sep 2023 12:06:29 +0200 Subject: [PATCH 1/2] feat: allow disabling jlap --- Cargo.toml | 2 +- .../src/fetch/jlap/mod.rs | 17 +++++++++++++++-- .../rattler_repodata_gateway/src/fetch/mod.rs | 19 ++++++++++++++++--- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4587edbc9..86edb1e5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,4 +23,4 @@ consolidate-commits = true tag-prefix = "" [profile.bench] -lto = true \ No newline at end of file +lto = true diff --git a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs index 35056eb2b..f59281058 100644 --- a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs @@ -400,6 +400,7 @@ pub async fn patch_repo_data( // We already have the latest version; return early because there's nothing to do if latest_hash == hash { + tracing::info!("The latest hash matches our local data. File up to date."); return Ok(jlap.get_state(jlap.new_position, new_iv)); } @@ -440,11 +441,15 @@ async fn fetch_jlap_with_retry( client: &AuthenticatedClient, position: u64, ) -> Result<(Response, u64), JLAPError> { - let range = format!("bytes={}-", position); + tracing::info!("fetching JLAP state from {url} (bytes={position}-)"); + let range = format!("bytes={position}-"); match fetch_jlap(url, client, &range).await { Ok(response) => { if response.status() == StatusCode::RANGE_NOT_SATISFIABLE && position != 0 { + tracing::warn!( + "JLAP range request could not be satisfied, fetching the entire file.." + ); let range = "bytes=0-"; return match fetch_jlap(url, client, range).await { Ok(response) => Ok((response, 0)), @@ -513,18 +518,25 @@ async fn apply_jlap_patches( // All these JSON operations are pretty CPU intensive therefor we move them to a blocking task // to ensure any other async operations will continue to purr along. let content = match tokio::task::spawn_blocking(move || { + tracing::info!("parsing cached repodata.json as JSON"); let mut doc = match serde_json::from_str(&repo_data_contents) { Ok(doc) => doc, Err(error) => return Err(JLAPError::JSONParse(error)), }; + tracing::info!( + "applying patches #{} through #{}", + start_index + 1, + patches.len() + ); // Apply the patches we current have to it - for patch in &patches[start_index..] { + for patch in patches[start_index..].iter() { if let Err(error) = json_patch::patch(&mut doc, &patch.patch) { return Err(JLAPError::JSONPatch(error)); } } + tracing::info!("converting patched JSON back to repodata"); let ordered_doc: OrderedRepoData = match serde_json::from_value(doc) { Ok(value) => value, Err(error) => return Err(JLAPError::JSONParse(error)), @@ -552,6 +564,7 @@ async fn apply_jlap_patches( } }; + tracing::info!("writing patched repodata to disk"); match tokio::fs::write(repo_data_path, &content).await { Ok(_) => Ok(()), Err(error) => Err(JLAPError::FileSystem(error)), diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index d8002ccb2..98b29fc1d 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -139,7 +139,7 @@ impl Variant { } /// Additional knobs that allow you to tweak the behavior of [`fetch_repo_data`]. -#[derive(Default, Clone)] +#[derive(Clone)] pub struct FetchRepoDataOptions { /// How to use the cache. By default it will cache and reuse downloaded repodata.json (if the /// server allows it). @@ -147,6 +147,19 @@ pub struct FetchRepoDataOptions { /// Determines which variant to download. See [`Variant`] for more information. pub variant: Variant, + + /// True if repodata can be fetch incrementally using JLAP. + pub jlap_enabled: bool, +} + +impl Default for FetchRepoDataOptions { + fn default() -> Self { + Self { + cache_action: Default::default(), + variant: Variant::default(), + jlap_enabled: true, + } + } } /// A struct that provides information about download progress. @@ -380,7 +393,7 @@ pub async fn fetch_repo_data( // We first attempt to make a JLAP request; if it fails for any reason, we continue on with // a normal request. - let jlap_state = if has_jlap && cache_state.is_some() { + let jlap_state = if has_jlap && cache_state.is_some() && options.jlap_enabled { let repo_data_state = cache_state.as_ref().unwrap(); match jlap::patch_repo_data( &client, @@ -391,7 +404,7 @@ pub async fn fetch_repo_data( .await { Ok(state) => { - tracing::debug!("fetched JLAP patches successfully"); + tracing::info!("fetched JLAP patches successfully"); let cache_state = RepoDataState { blake2_hash: Some(state.footer.latest), has_zst: variant_availability.has_zst, From e72c44426154150c7c501c58e6734b0cf9164b64 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Mon, 11 Sep 2023 13:55:53 +0200 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Tim de Jager --- crates/rattler_repodata_gateway/src/fetch/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index 98b29fc1d..6e1f97caa 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -148,7 +148,7 @@ pub struct FetchRepoDataOptions { /// Determines which variant to download. See [`Variant`] for more information. pub variant: Variant, - /// True if repodata can be fetch incrementally using JLAP. + /// When enabled repodata can be fetched incrementally using JLAP pub jlap_enabled: bool, }