Support sparse indexes

crate_universe/private/srcs.bzl

@@ -29,6 +29,7 @@ CARGO_BAZEL_SRCS = [
     Label("//crate_universe:src/rendering/templates/vendor_module.j2"),
     Label("//crate_universe:src/splicing.rs"),
     Label("//crate_universe:src/splicing/cargo_config.rs"),
+    Label("//crate_universe:src/splicing/crate_index_lookup.rs"),
     Label("//crate_universe:src/splicing/splicer.rs"),
     Label("//crate_universe:src/test.rs"),
     Label("//crate_universe:src/utils.rs"),

crate_universe/src/cli/splice.rs

@@ -98,9 +98,11 @@ pub fn splice(opt: SpliceOptions) -> Result<()> {
     )?;
     // Write the registry url info to the manifest now that a lockfile has been generated
     WorkspaceMetadata::write_registry_urls_and_feature_map(
+        &cargo,
         &cargo_lockfile,
         feature_map,
-        &manifest_path,
+        manifest_path.as_path_buf(),
+        manifest_path.as_path_buf(),
     )?;
 
     let output_dir = opt.output_dir.clone();

crate_universe/src/cli/vendor.rs

@@ -151,9 +151,11 @@ pub fn vendor(opt: VendorOptions) -> Result<()> {
 
     // Write the registry url info to the manifest now that a lockfile has been generated
     WorkspaceMetadata::write_registry_urls_and_feature_map(
+        &cargo,
         &cargo_lockfile,
         feature_map,
-        &manifest_path,
+        manifest_path.as_path_buf(),
+        manifest_path.as_path_buf(),
     )?;
 
     // Write metadata to the workspace for future reuse

crate_universe/src/metadata.rs

@@ -16,6 +16,7 @@ use crate::lockfile::Digest;
 use anyhow::{anyhow, bail, Context, Result};
 use cargo_lock::Lockfile as CargoLockfile;
 use cargo_metadata::{Metadata as CargoMetadata, MetadataCommand};
+use semver::Version;
 
 use crate::config::CrateId;
 use crate::utils::starlark::SelectList;
@@ -75,7 +76,7 @@ impl MetadataGenerator for Generator {
 
         let metadata = self
             .cargo_bin
-            .metadata_command()
+            .metadata_command()?
             .current_dir(manifest_dir)
             .manifest_path(manifest_path.as_ref())
             .other_options(["--locked".to_owned()])
@@ -103,15 +104,20 @@ impl Cargo {
     }
 
     /// Returns a new `Command` for running this cargo.
-    pub fn command(&self) -> Command {
-        Command::new(&self.path)
+    pub fn command(&self) -> Result<Command> {
+        let mut command = Command::new(&self.path);
+        command.envs(self.env()?);
+        Ok(command)
     }
 
     /// Returns a new `MetadataCommand` using this cargo.
-    pub fn metadata_command(&self) -> MetadataCommand {
+    pub fn metadata_command(&self) -> Result<MetadataCommand> {
         let mut command = MetadataCommand::new();
         command.cargo_path(&self.path);
-        command
+        for (k, v) in self.env()? {
+            command.env(k, v);
+        }
+        Ok(command)
     }
 
     /// Returns the output of running `cargo version`, trimming any leading or trailing whitespace.
@@ -124,6 +130,28 @@ impl Cargo {
         }
         Ok(full_version.clone().unwrap())
     }
+
+    pub fn use_sparse_registries_for_crates_io(&self) -> Result<bool> {
+        let full_version = self.full_version()?;
+        let version_str = full_version.split(' ').nth(1);
+        if let Some(version_str) = version_str {
+            let version = Version::parse(version_str).context("Failed to parse cargo version")?;
+            return Ok(version.major >= 1 && version.minor >= 68);
+        }
+        bail!("Couldn't parse cargo version");
+    }
+
+    fn env(&self) -> Result<BTreeMap<String, String>> {
+        let mut map = BTreeMap::new();
+
+        if self.use_sparse_registries_for_crates_io()? {
+            map.insert(
+                "CARGO_REGISTRIES_CRATES_IO_PROTOCOL".into(),
+                "sparse".into(),
+            );
+        }
+        Ok(map)
+    }
 }
 
 /// A configuration desrcibing how to invoke [cargo update](https://doc.rust-lang.org/cargo/commands/cargo-update.html).
@@ -192,7 +220,7 @@ impl CargoUpdateRequest {
 
         // Simply invoke `cargo update`
         let output = cargo_bin
-            .command()
+            .command()?
             // Cargo detects config files based on `pwd` when running so
             // to ensure user provided Cargo config files are used, it's
             // critical to set the working directory to the manifest dir.
@@ -267,7 +295,7 @@ impl LockGenerator {
             // of having just generated a new one
             let output = self
                 .cargo_bin
-                .command()
+                .command()?
                 // Cargo detects config files based on `pwd` when running so
                 // to ensure user provided Cargo config files are used, it's
                 // critical to set the working directory to the manifest dir.
@@ -295,7 +323,7 @@ impl LockGenerator {
             // Simply invoke `cargo generate-lockfile`
             let output = self
                 .cargo_bin
-                .command()
+                .command()?
                 // Cargo detects config files based on `pwd` when running so
                 // to ensure user provided Cargo config files are used, it's
                 // critical to set the working directory to the manifest dir.
@@ -347,7 +375,7 @@ impl VendorGenerator {
         // Simply invoke `cargo generate-lockfile`
         let output = self
             .cargo_bin
-            .command()
+            .command()?
             // Cargo detects config files based on `pwd` when running so
             // to ensure user provided Cargo config files are used, it's
             // critical to set the working directory to the manifest dir.
@@ -410,7 +438,7 @@ impl FeatureGenerator {
             // - https://github.com/bazelbuild/rules_rust/issues/1662
             let output = self
                 .cargo_bin
-                .command()
+                .command()?
                 .current_dir(manifest_dir)
                 .arg("tree")
                 .arg("--locked")

crate_universe/src/splicing.rs

@@ -1,6 +1,7 @@
 //! This module is responsible for finding a Cargo workspace
 
 pub(crate) mod cargo_config;
+mod crate_index_lookup;
 mod splicer;
 
 use std::collections::{BTreeMap, BTreeSet};
@@ -9,16 +10,17 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
-use anyhow::{bail, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use cargo_toml::Manifest;
-use hex::ToHex;
 use serde::{Deserialize, Serialize};
 
 use crate::config::CrateId;
 use crate::metadata::{Cargo, CargoUpdateRequest, LockGenerator};
+use crate::utils;
 use crate::utils::starlark::{Label, SelectList};
 
 use self::cargo_config::CargoConfig;
+use self::crate_index_lookup::CrateIndexLookup;
 pub use self::splicer::*;
 
 type DirectPackageManifest = BTreeMap<String, cargo_toml::DependencyDetail>;
@@ -242,11 +244,13 @@ impl WorkspaceMetadata {
     }
 
     pub fn write_registry_urls_and_feature_map(
+        cargo: &Cargo,
         lockfile: &cargo_lock::Lockfile,
         features: BTreeMap<CrateId, SelectList<String>>,
-        manifest_path: &SplicedManifest,
+        input_manifest_path: &Path,
+        output_manifest_path: &Path,
     ) -> Result<()> {
-        let mut manifest = read_manifest(manifest_path.as_path_buf())?;
+        let mut manifest = read_manifest(input_manifest_path)?;
 
         let mut workspace_metaata = WorkspaceMetadata::try_from(
             manifest
@@ -259,7 +263,7 @@ impl WorkspaceMetadata {
                 .clone(),
         )?;
 
-        // Locate all packages soruced from a registry
+        // Locate all packages sourced from a registry
         let pkg_sources: Vec<&cargo_lock::Package> = lockfile
             .packages
             .iter()
@@ -276,8 +280,7 @@ impl WorkspaceMetadata {
         // Load the cargo config
         let cargo_config = {
             // Note that this path must match the one defined in `splicing::setup_cargo_config`
-            let config_path = manifest_path
-                .as_path_buf()
+            let config_path = input_manifest_path
                 .parent()
                 .unwrap()
                 .join(".cargo")
@@ -294,80 +297,77 @@ impl WorkspaceMetadata {
         let crate_indexes = index_urls
             .into_iter()
             .map(|url| {
-                let index = {
-                    // Ensure the correct registry is mapped based on the give Cargo config.
-                    let index_url = if let Some(config) = &cargo_config {
-                        if let Some(source) = config.get_source_from_url(&url) {
-                            if let Some(replace_with) = &source.replace_with {
-                                if let Some(replacement) = config.get_registry_index_url_by_name(replace_with) {
-                                    replacement
-                                } else {
-                                    bail!("Tried to replace registry {} with registry named {} but didn't have metadata about the replacement", url, replace_with);
-                                }
-                            } else {
-                                &url
-                            }
-                        } else {
-                            &url
-                        }
-                    } else {
-                        &url
+                // Ensure the correct registry is mapped based on the give Cargo config.
+                let index_url = if let Some(config) = &cargo_config {
+                    config.resolve_replacement_url(&url)?
+                } else {
+                    &url
+                };
+                let index = if cargo.use_sparse_registries_for_crates_io()?
+                    && index_url == utils::CRATES_IO_INDEX_URL
+                {
+                    CrateIndexLookup::Http(crates_index::SparseIndex::from_url(
+                        "sparse+https://index.crates.io/",
+                    )?)
+                } else if index_url.starts_with("sparse+https://") {
+                    CrateIndexLookup::Http(crates_index::SparseIndex::from_url(index_url)?)
+                } else {
+                    let index = {
+                        // Load the index for the current url
+                        let index =
+                            crates_index::Index::from_url(index_url).with_context(|| {
+                                format!("Failed to load index for url: {index_url}")
+                            })?;
+
+                        // Ensure each index has a valid index config
+                        index.index_config().with_context(|| {
+                            format!("`config.json` not found in index: {index_url}")
+                        })?;
+
+                        index
                     };
 
-                    // Load the index for the current url
-                    let index = crates_index::Index::from_url(index_url)
-                        .with_context(|| format!("Failed to load index for url: {index_url}"))?;
-
-                    // Ensure each index has a valid index config
-                    index.index_config().with_context(|| {
-                        format!("`config.json` not found in index: {index_url}")
-                    })?;
-
-                    index
+                    CrateIndexLookup::Git(index)
                 };
-
                 Ok((url, index))
             })
-            .collect::<Result<BTreeMap<String, crates_index::Index>>>()
+            .collect::<Result<BTreeMap<String, _>>>()
             .context("Failed to locate crate indexes")?;
 
         // Get the download URL of each package based on it's registry url.
         let additional_sources = pkg_sources
             .iter()
-            .filter_map(|pkg| {
+            .map(|pkg| {
                 let source_id = pkg.source.as_ref().unwrap();
-                let index = &crate_indexes[&source_id.url().to_string()];
-                let index_config = index.index_config().unwrap();
-
-                index.crate_(pkg.name.as_str()).map(|crate_idx| {
-                    crate_idx
-                        .versions()
-                        .iter()
-                        .find(|v| v.version() == pkg.version.to_string())
-                        .and_then(|v| {
-                            v.download_url(&index_config).map(|url| {
-                                let crate_id =
-                                    CrateId::new(v.name().to_owned(), v.version().to_owned());
-                                let sha256 = pkg
-                                    .checksum
-                                    .as_ref()
-                                    .and_then(|sum| {
-                                        sum.as_sha256().map(|sum| sum.encode_hex::<String>())
-                                    })
-                                    .unwrap_or_else(|| v.checksum().encode_hex::<String>());
-                                let source_info = SourceInfo { url, sha256 };
-                                (crate_id, source_info)
-                            })
-                        })
+                let source_url = source_id.url().to_string();
+                let lookup = crate_indexes.get(&source_url).ok_or_else(|| {
+                    anyhow!(
+                        "Couldn't find crate_index data for SourceID {:?}",
+                        source_id
+                    )
+                })?;
+                lookup.get_source_info(pkg).map(|source_info| {
+                    (
+                        CrateId::new(pkg.name.as_str().to_owned(), pkg.version.to_string()),
+                        source_info,
+                    )
                 })
             })
-            .flatten();
-
-        workspace_metaata.sources.extend(additional_sources);
+            .collect::<Result<Vec<_>>>()?;
+
+        workspace_metaata
+            .sources
+            .extend(
+                additional_sources
+                    .into_iter()
+                    .filter_map(|(crate_id, source_info)| {
+                        source_info.map(|source_info| (crate_id, source_info))
+                    }),
+            );
         workspace_metaata.features = features;
         workspace_metaata.inject_into(&mut manifest)?;
 
-        write_root_manifest(manifest_path.as_path_buf(), manifest)?;
+        write_root_manifest(output_manifest_path, manifest)?;
 
         Ok(())
     }