Add HTTP support to SparseIndex

.gitattributes

@@ -0,0 +1 @@
+*.txt text eol=lf

.gitignore

@@ -1,3 +1,6 @@
 target
 Cargo.lock
 _test
+
+# May be created during tests, but should not be present
+tests/testdata/sparse_registry_cache/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/.cache/cr/at/crates-index

Cargo.toml

@@ -16,6 +16,7 @@ rust-version = "1.60"
 git2 = { version = ">=0.16, <=0.17", default-features = false }
 hex = { version = "0.4.3", features = ["serde"] }
 home = "0.5.4"
+http = { version = "0.2", optional = true }
 memchr = "2.5.0"
 rayon = { version = "1.7.0", optional = true }
 rustc-hash = "1.1.0"
@@ -39,6 +40,7 @@ https = ["git2/https"]
 parallel = ["dep:rayon"]
 vendored-openssl = ["git2/vendored-openssl"]
 ssh = ["git2/ssh"]
+sparse-http = ["http"]
 
 [badges]
 maintenance = { status = "actively-developed" }

src/bare_index.rs

@@ -220,7 +220,7 @@ impl Index {
     /// to read majority of crates, prefer the [`Index::crates()`] iterator.
     #[must_use]
     pub fn crate_(&self, name: &str) -> Option<Crate> {
-        let rel_path = crate::crate_name_to_relative_path(name)?;
+        let rel_path = crate::crate_name_to_relative_path(name, None)?;
 
         // Attempt to load the .cache/ entry first, this is purely an acceleration
         // mechanism and can fail for a few reasons that are non-fatal

src/config.rs

@@ -56,7 +56,7 @@ fn try_read_toml(path: &Path) -> Result<Option<toml::Value>, Error> {
         return Ok(None);
     }
 
-    let toml = toml::from_str(&std::fs::read_to_string(&path)?).map_err(Error::Toml)?;
+    let toml = toml::from_str(&std::fs::read_to_string(path)?).map_err(Error::Toml)?;
     Ok(Some(toml))
 }
 

src/lib.rs

@@ -314,10 +314,11 @@ fn crate_prefix(accumulator: &mut String, crate_name: &str, separator: char) ->
     Some(())
 }
 
-fn crate_name_to_relative_path(crate_name: &str) -> Option<String> {
+fn crate_name_to_relative_path(crate_name: &str, separator: Option<char>) -> Option<String> {
+    let separator = separator.unwrap_or(std::path::MAIN_SEPARATOR);
     let mut rel_path = String::with_capacity(crate_name.len() + 6);
-    crate_prefix(&mut rel_path, crate_name, std::path::MAIN_SEPARATOR)?;
-    rel_path.push(std::path::MAIN_SEPARATOR);
+    crate_prefix(&mut rel_path, crate_name, separator)?;
+    rel_path.push(separator);
     rel_path.extend(crate_name.as_bytes().iter().map(|c| c.to_ascii_lowercase() as char));
 
     Some(rel_path)
@@ -454,6 +455,28 @@ impl Crate {
         })
     }
 
+    /// Writes a cache entry to disk in the same format as cargo
+    #[cfg(feature = "sparse-http")]
+    pub(crate) fn write_cache_entry(&self, path: &Path, version: &str) -> io::Result<()> {
+        const CURRENT_CACHE_VERSION: u8 = 3;
+        const CURRENT_INDEX_FORMAT_VERSION: u32 = 2;
+
+        let mut v = Vec::new();
+        v.push(CURRENT_CACHE_VERSION);
+        v.extend_from_slice(&CURRENT_INDEX_FORMAT_VERSION.to_le_bytes());
+        v.extend_from_slice(version.as_bytes());
+        v.push(0);
+
+        for version in self.versions() {
+            v.extend_from_slice(version.version().as_bytes());
+            v.push(0);
+            v.append(&mut serde_json::to_vec(version).unwrap());
+            v.push(0);
+        }
+
+        std::fs::write(path, v)
+    }
+
     /// All versions of this crate sorted chronologically by date originally published
     ///
     /// Warning: may be yanked or duplicate