add -Zgitoxide=fetch feature toggle and implementation.

This allows to use `gitoxide` for all fetch operations, boosting performance
for fetching the `crates.io` index by a factor of 2.2x, while being consistent
on all platforms.

For trying it, nightly builds of `cargo` can specify `-Zgitoxide=fetch`.
It's also possible to set the `__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2=1` environment
variable (value matters), which is when `-Zgitoxide=none` can be used
to use `git2` instead.

Limitations
-----------
Note that what follows are current shortcomings that will be addressed in future PRs.

- it's likely that authentication around the `ssh` protocol will work differently in practice
  as it uses the `ssh` program.
- clones from `file://` based crates indices will need the `git` binary to serve the locatl repository.
- the progress bar shown when fetching doesn't work like the orgiinal, but should already feel 'faster'.

.github/workflows/main.yml

@@ -74,7 +74,7 @@ jobs:
           os: windows-latest
           rust: stable-msvc
           other: i686-pc-windows-msvc
-        - name: Windows x86_64 gnu nightly
+        - name: Windows x86_64 gnu nightly # runs out of space while trying to link the test suite
           os: windows-latest
           rust: nightly-gnu
           other: i686-pc-windows-gnu
@@ -102,7 +102,16 @@ jobs:
       if: "!contains(matrix.rust, 'stable')"
 
     - run: cargo test
-    # The testsuite generates a huge amount of data, and fetch-smoke-test was
+    - name: Clear intermediate test output
+      run: |
+          df -h
+          rm -rf target/tmp
+          df -h
+    - name: gitoxide tests (all git-related tests)
+      run: cargo test git
+      env:
+        __CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2: 1
+      # The testsuite generates a huge amount of data, and fetch-smoke-test was
     # running out of disk space.
     - name: Clear test output
       run: |
@@ -156,6 +165,18 @@ jobs:
     - run: rustup update stable && rustup default stable
     - run: cargo test --manifest-path crates/resolver-tests/Cargo.toml
 
+  test_gitoxide:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: rustup update --no-self-update stable && rustup default stable
+      - run: rustup target add i686-unknown-linux-gnu
+      - run: sudo apt update -y && sudo apt install gcc-multilib libsecret-1-0 libsecret-1-dev -y
+      - run: rustup component add rustfmt || echo "rustfmt not available"
+      - run: cargo test
+        env:
+          __CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2: 1
+
   build_std:
     runs-on: ubuntu-latest
     env:
@@ -196,7 +217,7 @@ jobs:
     permissions:
       contents: none
     name: bors build finished
-    needs: [docs, rustfmt, test, resolver, build_std]
+    needs: [docs, rustfmt, test, resolver, build_std, test_gitoxide]
     runs-on: ubuntu-latest
     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto-cargo'"
     steps:

Cargo.toml

@@ -30,6 +30,8 @@ filetime = "0.2.9"
 flate2 = { version = "1.0.3", default-features = false, features = ["zlib"] }
 git2 = "0.16.0"
 git2-curl = "0.17.0"
+gix = { version = "0.38.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree"] }
+gix-features-for-configuration-only = { version = "0.27.0", package = "gix-features", features = [ "parallel" ] }
 glob = "0.3.0"
 hex = "0.4"
 hmac = "0.12.1"

crates/cargo-test-support/src/git.rs

@@ -247,3 +247,10 @@ pub fn tag(repo: &git2::Repository, name: &str) {
         false
     ));
 }
+
+/// Returns true if gitoxide is globally activated.
+///
+/// That way, tests that normally use `git2` can transparently use `gitoxide`.
+pub fn cargo_uses_gitoxide() -> bool {
+    std::env::var_os("__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2").map_or(false, |value| value == "1")
+}

src/cargo/core/features.rs

@@ -716,6 +716,8 @@ unstable_cli_options!(
     doctest_xcompile: bool = ("Compile and run doctests for non-host target using runner config"),
     dual_proc_macros: bool = ("Build proc-macros for both the host and the target"),
     features: Option<Vec<String>>  = (HIDDEN),
+    gitoxide: Option<GitoxideFeatures> = ("Use gitoxide for the given git interactions, or all of them if no argument is given"),
+    jobserver_per_rustc: bool = (HIDDEN),
     minimal_versions: bool = ("Resolve minimal dependency versions instead of maximum"),
     direct_minimal_versions: bool = ("Resolve minimal dependency versions instead of maximum (direct dependencies only)"),
     mtime_on_use: bool = ("Configure Cargo to update the mtime of used files"),
@@ -827,6 +829,74 @@ where
     parse_check_cfg(crates.into_iter()).map_err(D::Error::custom)
 }
 
+#[derive(Debug, Copy, Clone, Default, Deserialize)]
+pub struct GitoxideFeatures {
+    /// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
+    pub fetch: bool,
+    /// When cloning the index, perform a shallow clone. Maintain shallowness upon subsequent fetches.
+    pub shallow_index: bool,
+    /// When cloning git dependencies, perform a shallow clone and maintain shallowness on subsequent fetches.
+    pub shallow_deps: bool,
+    /// Checkout git dependencies using `gitoxide` (submodules are still handled by git2 ATM, and filters
+    /// like linefeed conversions are unsupported).
+    pub checkout: bool,
+    /// A feature flag which doesn't have any meaning except for preventing
+    /// `__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2=1` builds to enable all safe `gitoxide` features.
+    /// That way, `gitoxide` isn't actually used even though it's enabled.
+    pub internal_use_git2: bool,
+}
+
+impl GitoxideFeatures {
+    fn all() -> Self {
+        GitoxideFeatures {
+            fetch: true,
+            shallow_index: true,
+            checkout: true,
+            shallow_deps: true,
+            internal_use_git2: false,
+        }
+    }
+
+    /// Features we deem safe for everyday use - typically true when all tests pass with them
+    /// AND they are backwards compatible.
+    fn safe() -> Self {
+        GitoxideFeatures {
+            fetch: true,
+            shallow_index: false,
+            checkout: true,
+            shallow_deps: false,
+            internal_use_git2: false,
+        }
+    }
+}
+
+fn parse_gitoxide(
+    it: impl Iterator<Item = impl AsRef<str>>,
+) -> CargoResult<Option<GitoxideFeatures>> {
+    let mut out = GitoxideFeatures::default();
+    let GitoxideFeatures {
+        fetch,
+        shallow_index,
+        checkout,
+        shallow_deps,
+        internal_use_git2,
+    } = &mut out;
+
+    for e in it {
+        match e.as_ref() {
+            "fetch" => *fetch = true,
+            "shallow-index" => *shallow_index = true,
+            "shallow-deps" => *shallow_deps = true,
+            "checkout" => *checkout = true,
+            "internal-use-git2" => *internal_use_git2 = true,
+            _ => {
+                bail!("unstable 'gitoxide' only takes `fetch`, 'shallow-index', 'shallow-deps' and 'checkout' as valid inputs")
+            }
+        }
+    }
+    Ok(Some(out))
+}
+
 fn parse_check_cfg(
     it: impl Iterator<Item = impl AsRef<str>>,
 ) -> CargoResult<Option<(bool, bool, bool, bool)>> {
@@ -879,6 +949,13 @@ impl CliUnstable {
         for flag in flags {
             self.add(flag, &mut warnings)?;
         }
+
+        if self.gitoxide.is_none()
+            && std::env::var_os("__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2")
+                .map_or(false, |value| value == "1")
+        {
+            self.gitoxide = GitoxideFeatures::safe().into();
+        }
         Ok(warnings)
     }
 
@@ -967,6 +1044,7 @@ impl CliUnstable {
             "doctest-xcompile" => self.doctest_xcompile = parse_empty(k, v)?,
             "doctest-in-workspace" => self.doctest_in_workspace = parse_empty(k, v)?,
             "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?,
+            "jobserver-per-rustc" => self.jobserver_per_rustc = parse_empty(k, v)?,
             "host-config" => self.host_config = parse_empty(k, v)?,
             "target-applies-to-host" => self.target_applies_to_host = parse_empty(k, v)?,
             "publish-timeout" => self.publish_timeout = parse_empty(k, v)?,

src/cargo/ops/mod.rs

@@ -52,7 +52,7 @@ mod cargo_uninstall;
 mod common_for_install_and_uninstall;
 mod fix;
 mod lockfile;
-mod registry;
+pub(crate) mod registry;
 mod resolve;
 pub mod tree;
 mod vendor;

src/cargo/ops/registry.rs

@@ -619,9 +619,6 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult<
         handle.useragent(&format!("cargo {}", version()))?;
     }
 
-    // Empty string accept encoding expands to the encodings supported by the current libcurl.
-    handle.accept_encoding("")?;
-
     fn to_ssl_version(s: &str) -> CargoResult<SslVersion> {
         let version = match s {
             "default" => SslVersion::Default,
@@ -631,13 +628,15 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult<
             "tlsv1.2" => SslVersion::Tlsv12,
             "tlsv1.3" => SslVersion::Tlsv13,
             _ => bail!(
-                "Invalid ssl version `{}`,\
-                 choose from 'default', 'tlsv1', 'tlsv1.0', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'.",
-                s
+                "Invalid ssl version `{s}`,\
+                 choose from 'default', 'tlsv1', 'tlsv1.0', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'."
             ),
         };
         Ok(version)
     }
+
+    // Empty string accept encoding expands to the encodings supported by the current libcurl.
+    handle.accept_encoding("")?;
     if let Some(ssl_version) = &http.ssl_version {
         match ssl_version {
             SslVersionConfig::Single(s) => {

src/cargo/sources/git/mod.rs

@@ -1,5 +1,10 @@
 pub use self::source::GitSource;
 pub use self::utils::{fetch, GitCheckout, GitDatabase, GitRemote};
 mod known_hosts;
+mod oxide;
 mod source;
 mod utils;
+
+pub mod fetch {
+    pub type Error = gix::env::collate::fetch::Error<gix::refspec::parse::Error>;
+}