Enable shallow clones and fetches for registry and git dependencies.

The implementation hinges on passing information about the kind of clone
and fetch to the `fetch()` method, which then configures the fetch accordingly.

Note that it doesn't differentiate between initial clones and fetches as
the shallow-ness of the repository is maintained nonetheless.

Cargo.toml

@@ -30,7 +30,7 @@ filetime = "0.2.9"
 flate2 = { version = "1.0.3", default-features = false, features = ["zlib"] }
 git2 = "0.16.0"
 git2-curl = "0.17.0"
-gix = { version = "0.41.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree"] }
+gix = { git = "https://github.com/Byron/gitoxide", branch = "main", version = "0.41.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree"] }
 gix-features-for-configuration-only = { version = "0.28.0", package = "gix-features", features = [ "parallel" ] }
 glob = "0.3.0"
 hex = "0.4"

src/cargo/sources/git/mod.rs

@@ -6,5 +6,14 @@ mod source;
 mod utils;
 
 pub mod fetch {
+    /// The kind remote repository to fetch.
+    #[derive(Debug, Copy, Clone)]
+    pub enum RemoteKind {
+        /// A repository belongs to a git dependency.
+        GitDependency,
+        /// A repository belongs to a Cargo registry.
+        Registry,
+    }
+
     pub type Error = gix::env::collate::fetch::Error<gix::refspec::parse::Error>;
 }

src/cargo/sources/git/utils.rs

@@ -1,7 +1,9 @@
 //! Utilities for handling git repositories, mainly around
 //! authentication/cloning.
 
+use crate::core::features::GitoxideFeatures;
 use crate::core::{GitReference, Verbosity};
+use crate::sources::git::fetch::RemoteKind;
 use crate::sources::git::oxide;
 use crate::sources::git::oxide::cargo_config_to_gitoxide_overrides;
 use crate::util::errors::CargoResult;
@@ -96,9 +98,16 @@ impl GitRemote {
         // if we can. If that can successfully load our revision then we've
         // populated the database with the latest version of `reference`, so
         // return that database and the rev we resolve to.
+        let remote_kind = RemoteKind::GitDependency;
         if let Some(mut db) = db {
-            fetch(&mut db.repo, self.url.as_str(), reference, cargo_config)
-                .context(format!("failed to fetch into: {}", into.display()))?;
+            fetch(
+                &mut db.repo,
+                self.url.as_str(),
+                reference,
+                cargo_config,
+                remote_kind,
+            )
+            .context(format!("failed to fetch into: {}", into.display()))?;
             match locked_rev {
                 Some(rev) => {
                     if db.contains(rev) {
@@ -121,8 +130,14 @@ impl GitRemote {
         }
         paths::create_dir_all(into)?;
         let mut repo = init(into, true)?;
-        fetch(&mut repo, self.url.as_str(), reference, cargo_config)
-            .context(format!("failed to clone into: {}", into.display()))?;
+        fetch(
+            &mut repo,
+            self.url.as_str(),
+            reference,
+            cargo_config,
+            remote_kind,
+        )
+        .context(format!("failed to clone into: {}", into.display()))?;
         let rev = match locked_rev {
             Some(rev) => rev,
             None => reference.resolve(&repo)?,
@@ -432,7 +447,14 @@ impl<'a> GitCheckout<'a> {
             cargo_config
                 .shell()
                 .status("Updating", format!("git submodule `{}`", url))?;
-            fetch(&mut repo, &url, &reference, cargo_config).with_context(|| {
+            fetch(
+                &mut repo,
+                &url,
+                &reference,
+                cargo_config,
+                RemoteKind::GitDependency,
+            )
+            .with_context(|| {
                 format!(
                     "failed to fetch submodule `{}` from {}",
                     child.name().unwrap_or(""),
@@ -803,11 +825,14 @@ pub fn with_fetch_options(
     })
 }
 
+/// Note that `kind` is only needed to know how to interpret `gitoxide` feature options to potentially shallow-clone
+/// the repository.
 pub fn fetch(
     repo: &mut git2::Repository,
     orig_url: &str,
     reference: &GitReference,
     config: &Config,
+    kind: RemoteKind,
 ) -> CargoResult<()> {
     if config.frozen() {
         anyhow::bail!(
@@ -893,6 +918,21 @@ pub fn fetch(
         let git2_repo = repo;
         let config_overrides = cargo_config_to_gitoxide_overrides(config)?;
         let repo_reinitialized = AtomicBool::default();
+        let has_feature = |cb: &dyn Fn(GitoxideFeatures) -> bool| {
+            config
+                .cli_unstable()
+                .gitoxide
+                .map_or(false, |features| cb(features))
+        };
+        let shallow = match kind {
+            RemoteKind::GitDependency if has_feature(&|git| git.shallow_deps) => {
+                gix::remote::fetch::Shallow::DepthAtRemote(1.try_into().expect("non-zero"))
+            }
+            RemoteKind::Registry if has_feature(&|git| git.shallow_index) => {
+                gix::remote::fetch::Shallow::DepthAtRemote(1.try_into().expect("non-zero"))
+            }
+            _ => gix::remote::fetch::Shallow::NoChange,
+        };
         let res = oxide::with_retry_and_progress(
             &git2_repo.path().to_owned(),
             config,
@@ -952,6 +992,7 @@ pub fn fetch(
                         );
                         let outcome = connection
                             .prepare_fetch(gix::remote::ref_map::Options::default())?
+                            .with_shallow(shallow.clone())
                             .receive(should_interrupt)?;
                         Ok(outcome)
                     });

src/cargo/sources/registry/remote.rs

@@ -1,5 +1,6 @@
 use crate::core::{GitReference, PackageId, SourceId};
 use crate::sources::git;
+use crate::sources::git::fetch::RemoteKind;
 use crate::sources::registry::download;
 use crate::sources::registry::MaybeLock;
 use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
@@ -300,8 +301,14 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> {
         // checkout.
         let url = self.source_id.url();
         let repo = self.repo.borrow_mut().unwrap();
-        git::fetch(repo, url.as_str(), &self.index_git_ref, self.config)
-            .with_context(|| format!("failed to fetch `{}`", url))?;
+        git::fetch(
+            repo,
+            url.as_str(),
+            &self.index_git_ref,
+            self.config,
+            RemoteKind::Registry,
+        )
+        .with_context(|| format!("failed to fetch `{}`", url))?;
 
         // Create a dummy file to record the mtime for when we updated the
         // index.

tests/testsuite/git.rs

@@ -9,6 +9,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::thread;
 
+use crate::git_gc::find_index;
 use cargo_test_support::git::cargo_uses_gitoxide;
 use cargo_test_support::paths::{self, CargoPathExt};
 use cargo_test_support::registry::Package;
@@ -1828,6 +1829,46 @@ fn fetch_downloads() {
     p.cargo("fetch").with_stdout("").run();
 }
 
+#[cargo_test]
+fn gitoxide_clone_registry_with_shallow_protocol_and_follow_up_fetch_maintains_shallowness(
+) -> anyhow::Result<()> {
+    // TODO(Seb): is there an easier way to trigger the registry clone? There are 50 tests that can trigger it,
+    //            this one seemed easy enough.
+    Package::new("bar", "1.0.0").publish();
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                bar = "1.0"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+    p.cargo("fetch")
+        .arg("-Zgitoxide=fetch,shallow-index")
+        .masquerade_as_nightly_cargo(&["unstable features must be available for -Z gitoxide"])
+        .run();
+
+    let repo = gix::open_opts(find_index(), gix::open::Options::isolated())?;
+    assert_eq!(
+        repo.rev_parse_single("origin/HEAD")?
+            .ancestors()
+            .all()?
+            .count(),
+        1,
+        "shallow clones always start at depth of 1 to minimize download size"
+    );
+
+    // TODO: trigger another fetch - adding another package and building again should do the trick
+
+    Ok(())
+}
+
 #[cargo_test]
 fn fetch_downloads_with_git2_first_then_with_gitoxide_and_vice_versa() {
     let bar = git::new("bar", |project| {