feat: wire up global config for jlap / zstd and bzip2 support in repodata

docs/advanced/global_configuration.md

@@ -55,6 +55,14 @@ authentication_override_file = "/path/to/your/override.json"
     "oci://ghcr.io/channel-mirrors/bioconda",
     "https://prefix.dev/bioconda",
 ]
+
+[repodata_options]
+# disable fetching of jlap, bz2 or zstd repodata files.
+# This should only be used for specific old versions of artifactory and other non-compliant
+# servers.
+disable_jlap = true  # don't try to download repodata.jlap
+disable_bzip2 = true # don't try to download repodata.json.bz2
+disable_zstd = true  # don't try to download repodata.json.zst
 ```
 
 ## Mirror configuration

src/cli/global/common.rs

@@ -182,9 +182,13 @@ pub(super) async fn get_client_and_sparse_repodata(
     IndexMap<(Channel, Platform), SparseRepoData>,
 )> {
     let authenticated_client = build_reqwest_clients(Some(config)).1;
-    let platform_sparse_repodata =
-        repodata::fetch_sparse_repodata(channels, [Platform::current()], &authenticated_client)
-            .await?;
+    let platform_sparse_repodata = repodata::fetch_sparse_repodata(
+        channels,
+        [Platform::current()],
+        &authenticated_client,
+        Some(config),
+    )
+    .await?;
     Ok((authenticated_client, platform_sparse_repodata))
 }
 

src/cli/search.rs

@@ -143,8 +143,15 @@ pub async fn execute(args: Args) -> miette::Result<()> {
         build_reqwest_clients(None).1
     };
 
-    let repo_data =
-        Arc::new(fetch_sparse_repodata(channels.iter(), [args.platform], &client).await?);
+    let repo_data = Arc::new(
+        fetch_sparse_repodata(
+            channels.iter(),
+            [args.platform],
+            &client,
+            project.as_ref().map(|p| p.config()),
+        )
+        .await?,
+    );
 
     // When package name filter contains * (wildcard), it will search and display a list of packages matching this filter
     if package_name_filter.contains('*') {

src/config.rs

@@ -88,6 +88,16 @@ pub struct ConfigCliPrompt {
     change_ps1: Option<bool>,
 }
 
+#[derive(Clone, Default, Debug, Deserialize)]
+pub struct RepodataConfig {
+    /// Disable JLAP compression for repodata.
+    pub disable_jlap: Option<bool>,
+    /// Disable bzip2 compression for repodata.
+    pub disable_bzip2: Option<bool>,
+    /// Disable zstd compression for repodata.
+    pub disable_zstd: Option<bool>,
+}
+
 #[derive(Clone, Default, Debug, Deserialize)]
 pub struct Config {
     #[serde(default)]
@@ -113,6 +123,9 @@ pub struct Config {
 
     #[serde(skip)]
     pub channel_config: ChannelConfig,
+
+    /// Configuration for repodata fetching.
+    pub repodata_config: Option<RepodataConfig>,
 }
 
 impl From<ConfigCli> for Config {
@@ -157,7 +170,7 @@ impl Config {
                 tracing::info!("Loading global config from {}", location.display());
                 let global_config = fs::read_to_string(&location).unwrap_or_default();
                 if let Ok(config) = Config::from_toml(&global_config, &location) {
-                    merged_config.merge_config(&config);
+                    merged_config = merged_config.merge_config(config);
                 } else {
                     tracing::warn!(
                         "Could not load global config (invalid toml): {}",
@@ -173,16 +186,13 @@ impl Config {
         // This will add any environment variables defined in the `clap` attributes to the config
         let mut default_cli = ConfigCli::default();
         default_cli.update_from(std::env::args().take(0));
-        merged_config.merge_config(&default_cli.into());
-
-        merged_config
+        merged_config.merge_config(default_cli.into())
     }
 
     /// Load the global config and layer the given cli config on top of it.
     pub fn with_cli_config(cli: &ConfigCli) -> Config {
-        let mut config = Config::load_global();
-        config.merge_config(&cli.clone().into());
-        config
+        let config = Config::load_global();
+        config.merge_config(cli.clone().into())
     }
 
     /// Load the config from the given path pixi folder and merge it with the global config.
@@ -193,7 +203,7 @@ impl Config {
         if local_config.exists() {
             let s = fs::read_to_string(&local_config).into_diagnostic()?;
             let local = Config::from_toml(&s, &local_config)?;
-            config.merge_config(&local);
+            config = config.merge_config(local);
         }
 
         Ok(config)
@@ -205,26 +215,28 @@ impl Config {
     }
 
     /// Merge the given config into the current one.
-    pub fn merge_config(&mut self, other: &Config) {
-        if !other.default_channels.is_empty() {
-            self.default_channels = other.default_channels.clone();
-        }
+    #[must_use]
+    pub fn merge_config(mut self, other: Config) -> Self {
+        self.mirrors.extend(other.mirrors);
+        self.loaded_from.extend(other.loaded_from);
 
-        if other.change_ps1.is_some() {
-            self.change_ps1 = other.change_ps1;
-        }
-
-        if other.tls_no_verify.is_some() {
-            self.tls_no_verify = other.tls_no_verify;
-        }
-
-        if other.authentication_override_file.is_some() {
-            self.authentication_override_file = other.authentication_override_file.clone();
+        Self {
+            default_channels: if other.default_channels.is_empty() {
+                self.default_channels
+            } else {
+                other.default_channels
+            },
+            tls_no_verify: other.tls_no_verify.or(self.tls_no_verify),
+            change_ps1: other.change_ps1.or(self.change_ps1),
+            authentication_override_file: other
+                .authentication_override_file
+                .or(self.authentication_override_file),
+            mirrors: self.mirrors,
+            loaded_from: self.loaded_from,
+            // currently this is always the default so just use the current value
+            channel_config: self.channel_config,
+            repodata_config: other.repodata_config.or(self.repodata_config),
         }
-
-        self.mirrors.extend(other.mirrors.clone());
-
-        self.loaded_from.extend(other.loaded_from.iter().cloned());
     }
 
     /// Retrieve the value for the default_channels field (defaults to the ["conda-forge"]).
@@ -324,7 +336,7 @@ mod tests {
             tls_no_verify: Some(true),
             ..Default::default()
         };
-        config.merge_config(&other);
+        config = config.merge_config(other);
         assert_eq!(config.default_channels, vec!["conda-forge"]);
         assert_eq!(config.tls_no_verify, Some(true));
 
@@ -336,7 +348,7 @@ mod tests {
         let config_2 = Config::from_path(&d.join("config_2.toml")).unwrap();
 
         let mut merged = config_1.clone();
-        merged.merge_config(&config_2);
+        merged = merged.merge_config(config_2);
 
         let debug = format!("{:#?}", merged);
         let debug = debug.replace("\\\\", "/");

src/lock_file/update.rs

@@ -470,6 +470,7 @@ pub async fn ensure_up_to_date_lock_file(
             .into_iter()
             .filter(|target| !options.existing_repo_data.contains_key(target)),
         project.authenticated_client(),
+        Some(project.config()),
     )
     .await?;
 

src/project/mod.rs

@@ -222,7 +222,7 @@ impl Project {
     where
         C: Into<Config>,
     {
-        self.config.merge_config(&config.into());
+        self.config = self.config.merge_config(config.into());
         self
     }
 

src/repodata.rs

@@ -1,3 +1,4 @@
+use crate::config::Config;
 use crate::project::Environment;
 use crate::{config, progress, project::Project};
 use futures::{stream, StreamExt, TryStreamExt};
@@ -26,14 +27,21 @@ impl Environment<'_> {
     ) -> miette::Result<IndexMap<(Channel, Platform), SparseRepoData>> {
         let channels = self.channels();
         let platforms = self.platforms();
-        fetch_sparse_repodata(channels, platforms, self.project().authenticated_client()).await
+        fetch_sparse_repodata(
+            channels,
+            platforms,
+            self.project().authenticated_client(),
+            Some(self.project().config()),
+        )
+        .await
     }
 }
 
 pub async fn fetch_sparse_repodata(
     channels: impl IntoIterator<Item = &'_ Channel>,
     target_platforms: impl IntoIterator<Item = Platform>,
     authenticated_client: &ClientWithMiddleware,
+    config: Option<&Config>,
 ) -> miette::Result<IndexMap<(Channel, Platform), SparseRepoData>> {
     let channels = channels.into_iter();
     let target_platforms = target_platforms.into_iter().collect_vec();
@@ -54,12 +62,13 @@ pub async fn fetch_sparse_repodata(
         }
     }
 
-    fetch_sparse_repodata_targets(fetch_targets, authenticated_client).await
+    fetch_sparse_repodata_targets(fetch_targets, authenticated_client, config).await
 }
 
 pub async fn fetch_sparse_repodata_targets(
     fetch_targets: impl IntoIterator<Item = (Channel, Platform)>,
     authenticated_client: &ClientWithMiddleware,
+    config: Option<&Config>,
 ) -> miette::Result<IndexMap<(Channel, Platform), SparseRepoData>> {
     let mut fetch_targets = fetch_targets.into_iter().peekable();
     if fetch_targets.peek().is_none() {
@@ -78,6 +87,17 @@ pub async fn fetch_sparse_repodata_targets(
     let multi_progress = progress::global_multi_progress();
     let mut progress_bars = Vec::new();
 
+    let fetch_repodata_options = config
+        .as_ref()
+        .and_then(|config| config.repodata_config.as_ref())
+        .map(|config| FetchRepoDataOptions {
+            bz2_enabled: !config.disable_bzip2.unwrap_or_default(),
+            jlap_enabled: !config.disable_jlap.unwrap_or_default(),
+            zstd_enabled: !config.disable_zstd.unwrap_or_default(),
+            ..Default::default()
+        })
+        .unwrap_or_default();
+
     let repo_data = stream::iter(fetch_targets)
         .map(|(channel, platform)| {
             // Construct a progress bar for the fetch
@@ -93,7 +113,7 @@ pub async fn fetch_sparse_repodata_targets(
             let repodata_cache = repodata_cache_path.clone();
             let download_client = authenticated_client.clone();
             let top_level_progress = top_level_progress.clone();
-
+            let fetch_options = fetch_repodata_options.clone();
             async move {
                 let result = fetch_repo_data_records_with_progress(
                     channel.clone(),
@@ -102,6 +122,7 @@ pub async fn fetch_sparse_repodata_targets(
                     download_client,
                     progress_bar.clone(),
                     platform != Platform::NoArch,
+                    fetch_options,
                 )
                 .await;
 
@@ -133,16 +154,15 @@ async fn fetch_repo_data_records_with_progress(
     client: ClientWithMiddleware,
     progress_bar: indicatif::ProgressBar,
     allow_not_found: bool,
+    fetch_options: FetchRepoDataOptions,
 ) -> miette::Result<Option<SparseRepoData>> {
     // Download the repodata.json
     let download_progress_progress_bar = progress_bar.clone();
     let result = fetch::fetch_repo_data(
         channel.platform_url(platform),
         client,
         repodata_cache.to_path_buf(),
-        FetchRepoDataOptions {
-            ..FetchRepoDataOptions::default()
-        },
+        fetch_options,
         Some(Box::new(move |fetch::DownloadProgress { total, bytes }| {
             download_progress_progress_bar.set_length(total.unwrap_or(bytes));
             download_progress_progress_bar.set_position(bytes);

src/snapshots/pixi__config__tests__config_merge.snap

@@ -37,4 +37,15 @@ Config {
             fragment: None,
         },
     },
+    repodata_config: Some(
+        RepodataConfig {
+            disable_jlap: Some(
+                true,
+            ),
+            disable_bzip2: None,
+            disable_zstd: Some(
+                true,
+            ),
+        },
+    ),
 }

tests/config/config_2.toml

@@ -1,2 +1,6 @@
 tls_no_verify = false
 change_ps1 = true
+
+[repodata_config]
+disable_jlap = true
+disable_zstd = true