add -Zgitoxide=fetch feature toggle and implementation.

This allows to use `gitoxide` for all fetch operations, boosting performance
for fetching the `crates.io` index by a factor of 2.2x, while being consistent
on all platforms.

For trying it, nightly builds of `cargo` can specify `-Zgitoxide=fetch`.
It's also possible to set the `__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2=1` environment
variable (value matters), which is when `-Zgitoxide=none` can be used
to use `git2` instead.

Limitations
-----------
Note that what follows are current shortcomings that will be addressed in future PRs.

- it's likely that authentication around the `ssh` protocol will work differently in practice
  as it uses the `ssh` program.
- clones from `file://` based crates indices will need the `git` binary to serve the locatl repository.
- the progress bar shown when fetching doesn't work like the orgiinal, but should already feel 'faster'.

.github/workflows/main.yml

@@ -74,7 +74,7 @@ jobs:
  os: windows-latest
  rust: stable-msvc
  other: i686-pc-windows-msvc
- - name: Windows x86_64 gnu nightly
+ - name: Windows x86_64 gnu nightly # runs out of space while trying to link the test suite
  os: windows-latest
  rust: nightly-gnu
  other: i686-pc-windows-gnu
@@ -102,7 +102,16 @@ jobs:
  if: "!contains(matrix.rust, 'stable')"
 
  - run: cargo test
- # The testsuite generates a huge amount of data, and fetch-smoke-test was
+ - name: Clear intermediate test output
+ run: |
+ df -h
+ rm -rf target/tmp
+ df -h
+ - name: gitoxide tests (all git-related tests)
+ run: cargo test git
+ env:
+ __CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2: 1
+ # The testsuite generates a huge amount of data, and fetch-smoke-test was
  # running out of disk space.
  - name: Clear test output
  run: |
@@ -156,6 +165,18 @@ jobs:
  - run: rustup update stable && rustup default stable
  - run: cargo test --manifest-path crates/resolver-tests/Cargo.toml
 
+ test_gitoxide:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - run: rustup update --no-self-update stable && rustup default stable
+ - run: rustup target add i686-unknown-linux-gnu
+ - run: sudo apt update -y && sudo apt install gcc-multilib libsecret-1-0 libsecret-1-dev -y
+ - run: rustup component add rustfmt || echo "rustfmt not available"
+ - run: cargo test
+ env:
+ __CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2: 1
+
  build_std:
  runs-on: ubuntu-latest
  env:
@@ -196,7 +217,7 @@ jobs:
  permissions:
  contents: none
  name: bors build finished
- needs: [docs, rustfmt, test, resolver, build_std]
+ needs: [docs, rustfmt, test, resolver, build_std, test_gitoxide]
  runs-on: ubuntu-latest
  if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto-cargo'"
  steps:

Cargo.toml

@@ -30,6 +30,8 @@ filetime = "0.2.9"
 flate2 = { version = "1.0.3", default-features = false, features = ["zlib"] }
 git2 = "0.16.0"
 git2-curl = "0.17.0"
+gix = { version = "0.38.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree"] }
+gix-features-for-configuration-only = { version = "0.27.0", package = "gix-features", features = [ "parallel" ] }
 glob = "0.3.0"
 hex = "0.4"
 hmac = "0.12.1"

crates/cargo-test-support/src/git.rs

@@ -247,3 +247,10 @@ pub fn tag(repo: &git2::Repository, name: &str) {
  false
  ));
 }
+
+/// Returns true if gitoxide is globally activated.
+///
+/// That way, tests that normally use `git2` can transparently use `gitoxide`.
+pub fn cargo_uses_gitoxide() -> bool {
+ std::env::var_os("__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2").map_or(false, |value| value == "1")
+}

src/cargo/core/features.rs

@@ -716,6 +716,8 @@ unstable_cli_options!(
  doctest_xcompile: bool = ("Compile and run doctests for non-host target using runner config"),
  dual_proc_macros: bool = ("Build proc-macros for both the host and the target"),
  features: Option<Vec<String>> = (HIDDEN),
+ gitoxide: Option<GitoxideFeatures> = ("Use gitoxide for the given git interactions, or all of them if no argument is given"),
+ jobserver_per_rustc: bool = (HIDDEN),
  minimal_versions: bool = ("Resolve minimal dependency versions instead of maximum"),
  direct_minimal_versions: bool = ("Resolve minimal dependency versions instead of maximum (direct dependencies only)"),
  mtime_on_use: bool = ("Configure Cargo to update the mtime of used files"),
@@ -827,6 +829,74 @@ where
  parse_check_cfg(crates.into_iter()).map_err(D::Error::custom)
 }
 
+#[derive(Debug, Copy, Clone, Default, Deserialize)]
+pub struct GitoxideFeatures {
+ /// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index.
+ pub fetch: bool,
+ /// When cloning the index, perform a shallow clone. Maintain shallowness upon subsequent fetches.
+ pub shallow_index: bool,
+ /// When cloning git dependencies, perform a shallow clone and maintain shallowness on subsequent fetches.
+ pub shallow_deps: bool,
+ /// Checkout git dependencies using `gitoxide` (submodules are still handled by git2 ATM, and filters
+ /// like linefeed conversions are unsupported).
+ pub checkout: bool,
+ /// A feature flag which doesn't have any meaning except for preventing
+ /// `__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2=1` builds to enable all safe `gitoxide` features.
+ /// That way, `gitoxide` isn't actually used even though it's enabled.
+ pub internal_use_git2: bool,
+}
+
+impl GitoxideFeatures {
+ fn all() -> Self {
+ GitoxideFeatures {
+ fetch: true,
+ shallow_index: true,
+ checkout: true,
+ shallow_deps: true,
+ internal_use_git2: false,
+ }
+ }
+
+ /// Features we deem safe for everyday use - typically true when all tests pass with them
+ /// AND they are backwards compatible.
+ fn safe() -> Self {
+ GitoxideFeatures {
+ fetch: true,
+ shallow_index: false,
+ checkout: true,
+ shallow_deps: false,
+ internal_use_git2: false,
+ }
+ }
+}
+
+fn parse_gitoxide(
+ it: impl Iterator<Item = impl AsRef<str>>,
+) -> CargoResult<Option<GitoxideFeatures>> {
+ let mut out = GitoxideFeatures::default();
+ let GitoxideFeatures {
+ fetch,
+ shallow_index,
+ checkout,
+ shallow_deps,
+ internal_use_git2,
+ } = &mut out;
+
+ for e in it {
+ match e.as_ref() {
+ "fetch" => *fetch = true,
+ "shallow-index" => *shallow_index = true,
+ "shallow-deps" => *shallow_deps = true,
+ "checkout" => *checkout = true,
+ "internal-use-git2" => *internal_use_git2 = true,
+ _ => {
+ bail!("unstable 'gitoxide' only takes `fetch`, 'shallow-index', 'shallow-deps' and 'checkout' as valid inputs")
+ }
+ }
+ }
+ Ok(Some(out))
+}
+
 fn parse_check_cfg(
  it: impl Iterator<Item = impl AsRef<str>>,
 ) -> CargoResult<Option<(bool, bool, bool, bool)>> {
@@ -879,6 +949,13 @@ impl CliUnstable {
  for flag in flags {
  self.add(flag, &mut warnings)?;
  }
+
+ if self.gitoxide.is_none()
+ && std::env::var_os("__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2")
+ .map_or(false, |value| value == "1")
+ {
+ self.gitoxide = GitoxideFeatures::safe().into();
+ }
  Ok(warnings)
  }
 
@@ -967,6 +1044,7 @@ impl CliUnstable {
  "doctest-xcompile" => self.doctest_xcompile = parse_empty(k, v)?,
  "doctest-in-workspace" => self.doctest_in_workspace = parse_empty(k, v)?,
  "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?,
+ "jobserver-per-rustc" => self.jobserver_per_rustc = parse_empty(k, v)?,
  "host-config" => self.host_config = parse_empty(k, v)?,
  "target-applies-to-host" => self.target_applies_to_host = parse_empty(k, v)?,
  "publish-timeout" => self.publish_timeout = parse_empty(k, v)?,

src/cargo/ops/mod.rs

@@ -52,7 +52,7 @@ mod cargo_uninstall;
 mod common_for_install_and_uninstall;
 mod fix;
 mod lockfile;
-mod registry;
+pub(crate) mod registry;
 mod resolve;
 pub mod tree;
 mod vendor;

src/cargo/ops/registry.rs

@@ -619,9 +619,6 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult<
  handle.useragent(&format!("cargo {}", version()))?;
  }
 
- // Empty string accept encoding expands to the encodings supported by the current libcurl.
- handle.accept_encoding("")?;
-
  fn to_ssl_version(s: &str) -> CargoResult<SslVersion> {
  let version = match s {
  "default" => SslVersion::Default,
@@ -631,13 +628,15 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult<
  "tlsv1.2" => SslVersion::Tlsv12,
  "tlsv1.3" => SslVersion::Tlsv13,
  _ => bail!(
- "Invalid ssl version `{}`,\
- choose from 'default', 'tlsv1', 'tlsv1.0', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'.",
- s
+ "Invalid ssl version `{s}`,\
+ choose from 'default', 'tlsv1', 'tlsv1.0', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'."
  ),
  };
  Ok(version)
  }
+
+ // Empty string accept encoding expands to the encodings supported by the current libcurl.
+ handle.accept_encoding("")?;
  if let Some(ssl_version) = &http.ssl_version {
  match ssl_version {
  SslVersionConfig::Single(s) => {

src/cargo/sources/git/mod.rs

@@ -1,5 +1,10 @@
 pub use self::source::GitSource;
 pub use self::utils::{fetch, GitCheckout, GitDatabase, GitRemote};
 mod known_hosts;
+mod oxide;
 mod source;
 mod utils;
+
+pub mod fetch {
+ pub type Error = gix::env::collate::fetch::Error<gix::refspec::parse::Error>;
+}