diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 2a4021dfd..d900d4bdb 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -22,12 +22,14 @@ dirs = "5.0.1" keyring = "2.0.5" lazy_static = "1.4.0" libc = "0.2.148" -reqwest = { version = "0.11.22", default-features = false} +reqwest = { version = "0.11.22", default-features = false } retry-policies = { version = "0.2.0", default-features = false } serde = "1.0.188" serde_json = "1.0.107" thiserror = "1.0.49" tracing = "0.1.37" +url = "2.4.1" +itertools = "0.11.0" [target.'cfg( target_arch = "wasm32" )'.dependencies] getrandom = { version = "0.2.10", features = ["js"] } diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index 152e557f1..526aae783 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -11,6 +11,12 @@ use reqwest::{Client, IntoUrl, Method, Url}; pub mod authentication_storage; pub mod retry_policies; +mod redaction; + +pub use redaction::{ + redact_known_secrets_from_error, redact_known_secrets_from_url, DEFAULT_REDACTION_STR, +}; + /// A client that can be used to make authenticated requests, based on the [`reqwest::Client`]. /// By default it uses the fallback storage in the default [`default_auth_store_fallback_directory`]. #[derive(Clone, Default)] diff --git a/crates/rattler_networking/src/redaction.rs b/crates/rattler_networking/src/redaction.rs new file mode 100644 index 000000000..812128ec8 --- /dev/null +++ b/crates/rattler_networking/src/redaction.rs @@ -0,0 +1,86 @@ +use itertools::Itertools; +use url::Url; + +/// A default string to use for redaction. +pub const DEFAULT_REDACTION_STR: &str = "xxxxxxxx"; + +/// Anaconda channels are not always publicly available. This function checks if a URL contains a +/// secret by identifying whether it contains certain patterns. If it does, the function returns a +/// modified URL where any secret has been masked. +/// +/// The `redaction` argument can be used to specify a custom string that should be used to replace +/// a secret. For consistency between application it is recommended to pass +/// [`DEFAULT_REDACTION_STR`]. +/// +/// # Example +/// +/// ```rust +/// # use rattler_networking::{redact_known_secrets_from_url, DEFAULT_REDACTION_STR}; +/// # use url::Url; +/// +/// let url = Url::parse("https://conda.anaconda.org/t/12345677/conda-forge/noarch/repodata.json").unwrap(); +/// let redacted_url = redact_known_secrets_from_url(&url, DEFAULT_REDACTION_STR).unwrap_or(url); +/// ``` +pub fn redact_known_secrets_from_url(url: &Url, redaction: &str) -> Option { + let mut segments = url.path_segments()?; + match (segments.next(), segments.next()) { + (Some("t"), Some(_)) => { + let remainder = segments.collect_vec(); + let redacted_path = format!( + "t/{redaction}{seperator}{remainder}", + seperator = if remainder.is_empty() { "" } else { "/" }, + remainder = remainder.iter().format("/") + ); + + let mut url = url.clone(); + url.set_path(&redacted_path); + Some(url) + } + _ => None, + } +} + +/// Redacts known secrets from a [`reqwest::Error`]. +pub fn redact_known_secrets_from_error(err: reqwest::Error) -> reqwest::Error { + if let Some(url) = err.url() { + let redacted_url = redact_known_secrets_from_url(url, DEFAULT_REDACTION_STR) + .unwrap_or_else(|| url.clone()); + err.with_url(redacted_url) + } else { + err + } +} + +#[cfg(test)] +mod test { + use super::*; + use std::str::FromStr; + + #[test] + fn test_remove_known_secrets_from_url() { + assert_eq!( + redact_known_secrets_from_url( + &Url::from_str( + "https://conda.anaconda.org/t/12345677/conda-forge/noarch/repodata.json" + ) + .unwrap(), + DEFAULT_REDACTION_STR + ), + Some( + Url::from_str( + "https://conda.anaconda.org/t/xxxxxxxx/conda-forge/noarch/repodata.json" + ) + .unwrap() + ) + ); + + assert_eq!( + redact_known_secrets_from_url( + &Url::from_str("https://conda.anaconda.org/conda-forge/noarch/repodata.json") + .unwrap(), + "helloworld" + ), + None, + ); + } +} diff --git a/crates/rattler_package_streaming/src/lib.rs b/crates/rattler_package_streaming/src/lib.rs index f851f9276..4bd75427e 100644 --- a/crates/rattler_package_streaming/src/lib.rs +++ b/crates/rattler_package_streaming/src/lib.rs @@ -3,6 +3,7 @@ //! This crate provides the ability to extract a Conda package archive or specific parts of it. use rattler_digest::{Md5Hash, Sha256Hash}; +use rattler_networking::redact_known_secrets_from_error; pub mod read; pub mod seek; @@ -45,6 +46,13 @@ pub enum ExtractError { Cancelled, } +#[cfg(feature = "reqwest")] +impl From<::reqwest::Error> for ExtractError { + fn from(err: ::reqwest::Error) -> Self { + Self::ReqwestError(redact_known_secrets_from_error(err)) + } +} + /// Result struct returned by extraction functions. #[derive(Debug)] pub struct ExtractResult { diff --git a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs index ef3a622bf..611f7917c 100644 --- a/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs @@ -80,7 +80,7 @@ use blake2::digest::{FixedOutput, Update}; use rattler_digest::{ parse_digest_from_hex, serde::SerializableHash, Blake2b256, Blake2b256Hash, Blake2bMac256, }; -use rattler_networking::AuthenticatedClient; +use rattler_networking::{redact_known_secrets_from_error, AuthenticatedClient}; use reqwest::{ header::{HeaderMap, HeaderValue}, Response, StatusCode, @@ -160,6 +160,12 @@ pub enum JLAPError { Cancelled, } +impl From for JLAPError { + fn from(value: reqwest::Error) -> Self { + Self::HTTP(redact_known_secrets_from_error(value)) + } +} + /// Represents the numerous patches found in a JLAP file which makes up a majority /// of the file #[serde_as] @@ -406,7 +412,7 @@ pub async fn patch_repo_data( fetch_jlap_with_retry(jlap_url.as_str(), client, jlap_state.position).await?; let response_text = match response.text().await { Ok(value) => value, - Err(error) => return Err(JLAPError::HTTP(error)), + Err(error) => return Err(error.into()), }; // Update position as it may have changed @@ -477,12 +483,12 @@ async fn fetch_jlap_with_retry( let range = "bytes=0-"; return match fetch_jlap(url, client, range).await { Ok(response) => Ok((response, 0)), - Err(error) => Err(JLAPError::HTTP(error)), + Err(error) => Err(error.into()), }; } Ok((response, position)) } - Err(error) => Err(JLAPError::HTTP(error)), + Err(error) => Err(error.into()), } } diff --git a/crates/rattler_repodata_gateway/src/fetch/mod.rs b/crates/rattler_repodata_gateway/src/fetch/mod.rs index 47f8964a5..ea32f89c1 100644 --- a/crates/rattler_repodata_gateway/src/fetch/mod.rs +++ b/crates/rattler_repodata_gateway/src/fetch/mod.rs @@ -6,7 +6,7 @@ use cache_control::{Cachability, CacheControl}; use futures::{future::ready, FutureExt, TryStreamExt}; use humansize::{SizeFormatter, DECIMAL}; use rattler_digest::{compute_file_digest, Blake2b256, HashingWriter}; -use rattler_networking::AuthenticatedClient; +use rattler_networking::{redact_known_secrets_from_error, AuthenticatedClient}; use reqwest::{ header::{HeaderMap, HeaderValue}, Response, StatusCode, @@ -32,7 +32,7 @@ pub type ProgressFunc = Box; pub enum RepoDataNotFoundError { /// There was an error on the Http request #[error(transparent)] - HttpError(#[from] reqwest::Error), + HttpError(reqwest::Error), /// There was a file system error #[error(transparent)] @@ -46,7 +46,7 @@ pub enum FetchRepoDataError { FailedToAcquireLock(#[source] anyhow::Error), #[error(transparent)] - HttpError(#[from] reqwest::Error), + HttpError(reqwest::Error), #[error(transparent)] FailedToDownloadRepoData(std::io::Error), @@ -73,6 +73,18 @@ pub enum FetchRepoDataError { Cancelled, } +impl From for FetchRepoDataError { + fn from(err: reqwest::Error) -> Self { + Self::HttpError(redact_known_secrets_from_error(err)) + } +} + +impl From for RepoDataNotFoundError { + fn from(value: reqwest::Error) -> Self { + Self::HttpError(redact_known_secrets_from_error(value)) + } +} + impl From for FetchRepoDataError { fn from(err: tokio::task::JoinError) -> Self { // Rethrow any panic