fix: redact tokens from urls in errors (#407)

This PR adds redaction of tokens in URLs used by errors. I may have
missed a few locations, but I think I got them all..

crates/rattler_networking/Cargo.toml

@@ -22,12 +22,14 @@ dirs = "5.0.1"
 keyring = "2.0.5"
 lazy_static = "1.4.0"
 libc = "0.2.148"
-reqwest = { version = "0.11.22", default-features = false}
+reqwest = { version = "0.11.22", default-features = false }
 retry-policies = { version = "0.2.0", default-features = false }
 serde = "1.0.188"
 serde_json = "1.0.107"
 thiserror = "1.0.49"
 tracing = "0.1.37"
+url = "2.4.1"
+itertools = "0.11.0"
 
 [target.'cfg( target_arch = "wasm32" )'.dependencies]
 getrandom = { version = "0.2.10", features = ["js"] }

crates/rattler_networking/src/lib.rs

@@ -11,6 +11,12 @@ use reqwest::{Client, IntoUrl, Method, Url};
 pub mod authentication_storage;
 pub mod retry_policies;
 
+mod redaction;
+
+pub use redaction::{
+    redact_known_secrets_from_error, redact_known_secrets_from_url, DEFAULT_REDACTION_STR,
+};
+
 /// A client that can be used to make authenticated requests, based on the [`reqwest::Client`].
 /// By default it uses the fallback storage in the default [`default_auth_store_fallback_directory`].
 #[derive(Clone, Default)]

crates/rattler_networking/src/redaction.rs

@@ -0,0 +1,86 @@
+use itertools::Itertools;
+use url::Url;
+
+/// A default string to use for redaction.
+pub const DEFAULT_REDACTION_STR: &str = "xxxxxxxx";
+
+/// Anaconda channels are not always publicly available. This function checks if a URL contains a
+/// secret by identifying whether it contains certain patterns. If it does, the function returns a
+/// modified URL where any secret has been masked.
+///
+/// The `redaction` argument can be used to specify a custom string that should be used to replace
+/// a secret. For consistency between application it is recommended to pass
+/// [`DEFAULT_REDACTION_STR`].
+///
+/// # Example
+///
+/// ```rust
+/// # use rattler_networking::{redact_known_secrets_from_url, DEFAULT_REDACTION_STR};
+/// # use url::Url;
+///
+/// let url = Url::parse("https://conda.anaconda.org/t/12345677/conda-forge/noarch/repodata.json").unwrap();
+/// let redacted_url = redact_known_secrets_from_url(&url, DEFAULT_REDACTION_STR).unwrap_or(url);
+/// ```
+pub fn redact_known_secrets_from_url(url: &Url, redaction: &str) -> Option<Url> {
+    let mut segments = url.path_segments()?;
+    match (segments.next(), segments.next()) {
+        (Some("t"), Some(_)) => {
+            let remainder = segments.collect_vec();
+            let redacted_path = format!(
+                "t/{redaction}{seperator}{remainder}",
+                seperator = if remainder.is_empty() { "" } else { "/" },
+                remainder = remainder.iter().format("/")
+            );
+
+            let mut url = url.clone();
+            url.set_path(&redacted_path);
+            Some(url)
+        }
+        _ => None,
+    }
+}
+
+/// Redacts known secrets from a [`reqwest::Error`].
+pub fn redact_known_secrets_from_error(err: reqwest::Error) -> reqwest::Error {
+    if let Some(url) = err.url() {
+        let redacted_url = redact_known_secrets_from_url(url, DEFAULT_REDACTION_STR)
+            .unwrap_or_else(|| url.clone());
+        err.with_url(redacted_url)
+    } else {
+        err
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::str::FromStr;
+
+    #[test]
+    fn test_remove_known_secrets_from_url() {
+        assert_eq!(
+            redact_known_secrets_from_url(
+                &Url::from_str(
+                    "https://conda.anaconda.org/t/12345677/conda-forge/noarch/repodata.json"
+                )
+                .unwrap(),
+                DEFAULT_REDACTION_STR
+            ),
+            Some(
+                Url::from_str(
+                    "https://conda.anaconda.org/t/xxxxxxxx/conda-forge/noarch/repodata.json"
+                )
+                .unwrap()
+            )
+        );
+
+        assert_eq!(
+            redact_known_secrets_from_url(
+                &Url::from_str("https://conda.anaconda.org/conda-forge/noarch/repodata.json")
+                    .unwrap(),
+                "helloworld"
+            ),
+            None,
+        );
+    }
+}

crates/rattler_package_streaming/src/lib.rs

@@ -3,6 +3,7 @@
 //! This crate provides the ability to extract a Conda package archive or specific parts of it.
 
 use rattler_digest::{Md5Hash, Sha256Hash};
+use rattler_networking::redact_known_secrets_from_error;
 
 pub mod read;
 pub mod seek;
@@ -45,6 +46,13 @@ pub enum ExtractError {
     Cancelled,
 }
 
+#[cfg(feature = "reqwest")]
+impl From<::reqwest::Error> for ExtractError {
+    fn from(err: ::reqwest::Error) -> Self {
+        Self::ReqwestError(redact_known_secrets_from_error(err))
+    }
+}
+
 /// Result struct returned by extraction functions.
 #[derive(Debug)]
 pub struct ExtractResult {

crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs

@@ -80,7 +80,7 @@ use blake2::digest::{FixedOutput, Update};
 use rattler_digest::{
     parse_digest_from_hex, serde::SerializableHash, Blake2b256, Blake2b256Hash, Blake2bMac256,
 };
-use rattler_networking::AuthenticatedClient;
+use rattler_networking::{redact_known_secrets_from_error, AuthenticatedClient};
 use reqwest::{
     header::{HeaderMap, HeaderValue},
     Response, StatusCode,
@@ -160,6 +160,12 @@ pub enum JLAPError {
     Cancelled,
 }
 
+impl From<reqwest::Error> for JLAPError {
+    fn from(value: reqwest::Error) -> Self {
+        Self::HTTP(redact_known_secrets_from_error(value))
+    }
+}
+
 /// Represents the numerous patches found in a JLAP file which makes up a majority
 /// of the file
 #[serde_as]
@@ -406,7 +412,7 @@ pub async fn patch_repo_data(
         fetch_jlap_with_retry(jlap_url.as_str(), client, jlap_state.position).await?;
     let response_text = match response.text().await {
         Ok(value) => value,
-        Err(error) => return Err(JLAPError::HTTP(error)),
+        Err(error) => return Err(error.into()),
     };
 
     // Update position as it may have changed
@@ -477,12 +483,12 @@ async fn fetch_jlap_with_retry(
                 let range = "bytes=0-";
                 return match fetch_jlap(url, client, range).await {
                     Ok(response) => Ok((response, 0)),
-                    Err(error) => Err(JLAPError::HTTP(error)),
+                    Err(error) => Err(error.into()),
                 };
             }
             Ok((response, position))
         }
-        Err(error) => Err(JLAPError::HTTP(error)),
+        Err(error) => Err(error.into()),
     }
 }
 

crates/rattler_repodata_gateway/src/fetch/mod.rs

@@ -6,7 +6,7 @@ use cache_control::{Cachability, CacheControl};
 use futures::{future::ready, FutureExt, TryStreamExt};
 use humansize::{SizeFormatter, DECIMAL};
 use rattler_digest::{compute_file_digest, Blake2b256, HashingWriter};
-use rattler_networking::AuthenticatedClient;
+use rattler_networking::{redact_known_secrets_from_error, AuthenticatedClient};
 use reqwest::{
     header::{HeaderMap, HeaderValue},
     Response, StatusCode,
@@ -32,7 +32,7 @@ pub type ProgressFunc = Box<dyn FnMut(DownloadProgress) + Send + Sync>;
 pub enum RepoDataNotFoundError {
     /// There was an error on the Http request
     #[error(transparent)]
-    HttpError(#[from] reqwest::Error),
+    HttpError(reqwest::Error),
 
     /// There was a file system error
     #[error(transparent)]
@@ -46,7 +46,7 @@ pub enum FetchRepoDataError {
     FailedToAcquireLock(#[source] anyhow::Error),
 
     #[error(transparent)]
-    HttpError(#[from] reqwest::Error),
+    HttpError(reqwest::Error),
 
     #[error(transparent)]
     FailedToDownloadRepoData(std::io::Error),
@@ -73,6 +73,18 @@ pub enum FetchRepoDataError {
     Cancelled,
 }
 
+impl From<reqwest::Error> for FetchRepoDataError {
+    fn from(err: reqwest::Error) -> Self {
+        Self::HttpError(redact_known_secrets_from_error(err))
+    }
+}
+
+impl From<reqwest::Error> for RepoDataNotFoundError {
+    fn from(value: reqwest::Error) -> Self {
+        Self::HttpError(redact_known_secrets_from_error(value))
+    }
+}
+
 impl From<tokio::task::JoinError> for FetchRepoDataError {
     fn from(err: tokio::task::JoinError) -> Self {
         // Rethrow any panic