diff --git a/Cargo.lock b/Cargo.lock index 8a3d916c6..673570156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,6 +418,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bisection" version = "0.1.0" @@ -887,7 +896,7 @@ dependencies = [ "hashbrown 0.14.3", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.9", ] [[package]] @@ -1160,7 +1169,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -1529,6 +1538,40 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-cache" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5ab65432bbdfe8490dfde21d0366353a8d39f2bc24aca0146889f931b0b4b5" +dependencies = [ + "async-trait", + "bincode", + "cacache", + "http", + "http-cache-semantics", + "httpdate", + "serde", + "url", +] + +[[package]] +name = "http-cache-reqwest" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8285341ce7e709c56a0f259ff1c789c70edfbaa88acd69d27e4d63980b92dc" +dependencies = [ + "anyhow", + "async-trait", + "http", + "http-cache", + "http-cache-semantics", + "reqwest", + "reqwest-middleware", + "serde", + "task-local-extensions", + "url", +] + [[package]] name = "http-cache-semantics" version = "1.0.2" @@ -1765,6 +1808,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -1917,7 +1963,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.2", "libc", - "redox_syscall", + "redox_syscall 0.4.1", ] [[package]] @@ -2451,6 +2497,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2458,7 +2515,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -2469,7 +2540,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -2691,6 +2762,7 @@ dependencies = [ "dunce", "flate2", "futures", + "http-cache-reqwest", "human_bytes", "humantime", "indexmap 2.2.2", @@ -2719,6 +2791,7 @@ dependencies = [ "regex", "reqwest", "reqwest-middleware", + "reqwest-retry", "rstest", "self-replace", "serde", @@ -3068,7 +3141,7 @@ dependencies = [ "miette 7.0.0", "mime", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "pathdiff", "peg", "pep440_rs", @@ -3302,6 +3375,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -3447,6 +3529,29 @@ dependencies = [ "thiserror", ] +[[package]] +name = "reqwest-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9af20b65c2ee9746cc575acb6bd28a05ffc0d15e25c992a8f4462d8686aacb4f" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "futures", + "getrandom", + "http", + "hyper", + "parking_lot 0.11.2", + "reqwest", + "reqwest-middleware", + "retry-policies", + "task-local-extensions", + "tokio", + "tracing", + "wasm-timer", +] + [[package]] name = "resolvo" version = "0.4.0" @@ -3849,7 +3954,7 @@ dependencies = [ "futures", "lazy_static", "log", - "parking_lot", + "parking_lot 0.12.1", "serial_test_derive", ] @@ -4773,6 +4878,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.68" diff --git a/Cargo.toml b/Cargo.toml index 20b652b5e..288c0f100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ dirs = "5.0.1" dunce = "1.0.4" flate2 = "1.0.28" futures = "0.3.30" +http-cache-reqwest = "0.13.0" human_bytes = "0.4.3" humantime = "2.1.0" indexmap = { version = "2.2.2", features = ["serde"] } @@ -56,6 +57,7 @@ rattler_virtual_packages = { version = "0.18.0", default-features = false } regex = "1.10.3" reqwest = { version = "0.11.24", default-features = false } reqwest-middleware = "0.2.4" +reqwest-retry = "0.3.0" rip = { package = "rattler_installs_packages", version = "0.8.1", default-features = false } self-replace = "1.3.7" serde = "1.0.196" diff --git a/src/lock_file/pypi_name_mapping.rs b/src/lock_file/pypi_name_mapping.rs index 6b470e017..eecfb8834 100644 --- a/src/lock_file/pypi_name_mapping.rs +++ b/src/lock_file/pypi_name_mapping.rs @@ -1,6 +1,12 @@ +use crate::config::get_cache_dir; use async_once_cell::OnceCell; +use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions}; use miette::{IntoDiagnostic, WrapErr}; use rattler_conda_types::{PackageUrl, RepoDataRecord}; +use rattler_networking::retry_policies::ExponentialBackoff; +use reqwest::Client; +use reqwest_middleware::ClientBuilder; +use reqwest_retry::RetryTransientMiddleware; use serde::Deserialize; use std::{collections::HashMap, str::FromStr}; use url::Url; @@ -15,7 +21,22 @@ struct CondaPyPiNameMapping { pub async fn conda_pypi_name_mapping() -> miette::Result<&'static HashMap> { static MAPPING: OnceCell> = OnceCell::new(); MAPPING.get_or_try_init(async { - let response = reqwest::get("https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/name_mapping.json").await + + // Construct a client with a retry policy and local caching + let retry_policy = + ExponentialBackoff::builder().build_with_max_retries(3); + let retry_strategy = RetryTransientMiddleware::new_with_policy(retry_policy); + let cache_strategy = Cache(HttpCache { + mode: CacheMode::Default, + manager: CACacheManager { path: get_cache_dir().expect("missing cache directory").join("http-cache") }, + options: HttpCacheOptions::default(), + }); + let client = ClientBuilder::new(Client::new()) + .with(cache_strategy) + .with(retry_strategy) + .build(); + + let response = client.get("https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/name_mapping.json").send().await .into_diagnostic() .context("failed to download pypi name mapping")?; let mapping: Vec = response