Auto merge of #13563 - ehuss:github-fast-path-token, r=<try>

Use GITHUB_TOKEN in github_fast_path if available.

This changes the GitHub fast-path to include an Authorization header if the GITHUB_TOKEN environment variable is set. This is intended to help with API rate limits, which is not too hard to hit in CI. If it hits the rate limit, then the fetch falls back to fetching all branches which is significantly slower for some repositories.

This is also a partial solution for #13555, where the user has a `rev="<commit-hash>"` where that commit hash points to a PR. The default refspec of `["+refs/heads/*:refs/remotes/origin/*", "+HEAD:refs/remotes/origin/HEAD"]` will not fetch commits from PRs (unless they are reachable from some branch).

There is some risk that if the user has a GITHUB_TOKEN that is invalid or expired, then this will cause `github_fast_path` to fail where it previously wouldn't. I think that is probably unlikely to happen, though.

This also adds redirect support. Apparently GitHub now issues a redirect for `repos/{org}/{repo}/commits/{commit}` to `/repositories/{id}/commits/{commit}`.

The test `https::github_works` exercises this code path, and will use the token on CI.

Author: bors

src/cargo/sources/git/utils.rs

@@ -1413,6 +1413,7 @@ fn github_fast_path(
                 // the branch has moved.
                 if let Some(local_object) = local_object {
                     if is_short_hash_of(rev, local_object) {
+                        debug!("github fast path already has {local_object}");
                         return Ok(FastPathRev::UpToDate);
                     }
                 }
@@ -1452,12 +1453,19 @@ fn github_fast_path(
     handle.get(true)?;
     handle.url(&url)?;
     handle.useragent("cargo")?;
+    handle.follow_location(true)?; // follow redirects
     handle.http_headers({
         let mut headers = List::new();
         headers.append("Accept: application/vnd.github.3.sha")?;
         if let Some(local_object) = local_object {
             headers.append(&format!("If-None-Match: \"{}\"", local_object))?;
         }
+        if let Ok(token) = gctx.get_env("GITHUB_TOKEN") {
+            if !token.is_empty() {
+                // Avoid API rate limits if possible.
+                headers.append(&format!("Authorization: Bearer {token}"))?;
+            }
+        }
         headers
     })?;
 
@@ -1472,14 +1480,17 @@ fn github_fast_path(
 
     let response_code = handle.response_code()?;
     if response_code == 304 {
+        debug!("github fast path up-to-date");
         Ok(FastPathRev::UpToDate)
     } else if response_code == 200 {
         let oid_to_fetch = str::from_utf8(&response_body)?.parse::<Oid>()?;
+        debug!("github fast path fetch {oid_to_fetch}");
         Ok(FastPathRev::NeedsFetch(oid_to_fetch))
     } else {
         // Usually response_code == 404 if the repository does not exist, and
         // response_code == 422 if exists but GitHub is unable to resolve the
         // requested rev.
+        debug!("github fast path bad response code {response_code}");
         Ok(FastPathRev::Indeterminate)
     }
 }

tests/testsuite/https.rs

@@ -4,6 +4,7 @@
 //! or CARGO_PUBLIC_NETWORK_TESTS.
 
 use cargo_test_support::containers::Container;
+use cargo_test_support::paths::{self, CargoPathExt};
 use cargo_test_support::project;
 
 #[cargo_test(container_test)]
@@ -134,22 +135,65 @@ fn self_signed_with_cacert() {
 #[cargo_test(public_network_test)]
 fn github_works() {
     // Check that an https connection to github.com works.
-    let p = project()
-        .file(
-            "Cargo.toml",
-            r#"
-                [package]
-                name = "foo"
-                version = "0.1.0"
-                edition = "2015"
+    // This tries all the different types of git references, and verifies the fast-path behavior.
+    for (manifest_ref, oid, refspecs, up_to_date) in [
+        (
+            r#", tag = "1.3.2""#,
+            "ed185cfb1c447c1b4bd6ac021c9ec3bb02c9e2f2",
+            r#""+refs/tags/1.3.2:refs/remotes/origin/tags/1.3.2""#,
+            "github fast path up-to-date",
+        ),
+        (
+            r#", rev = "6c67922300d5abae779ca147bac00f6ff9c87f8a""#,
+            "6c67922300d5abae779ca147bac00f6ff9c87f8a",
+            r#""+6c67922300d5abae779ca147bac00f6ff9c87f8a:refs/commit/6c67922300d5abae779ca147bac00f6ff9c87f8a""#,
+            "github fast path already has 6c67922300d5abae779ca147bac00f6ff9c87f8a",
+        ),
+        (
+            r#", branch = "main""#,
+            "[..]",
+            r#""+refs/heads/main:refs/remotes/origin/main""#,
+            "github fast path up-to-date",
+        ),
+        (
+            "",
+            "[..]",
+            r#""+HEAD:refs/remotes/origin/HEAD""#,
+            "github fast path up-to-date",
+        ),
+    ] {
+        eprintln!("test {manifest_ref}");
+        let p = project()
+            .file(
+                "Cargo.toml",
+                &format!(
+                    r#"
+                    [package]
+                    name = "foo"
+                    version = "0.1.0"
+                    edition = "2015"
 
-                [dependencies]
-                bitflags = { git = "https://github.com/rust-lang/bitflags.git", tag="1.3.2" }
-            "#,
-        )
-        .file("src/lib.rs", "")
-        .build();
-    p.cargo("fetch")
-        .with_stderr("[UPDATING] git repository `https://github.com/rust-lang/bitflags.git`")
-        .run();
+                    [dependencies]
+                    bitflags = {{ git = "https://github.com/rust-lang/bitflags.git"{manifest_ref}}}
+                "#
+                ),
+            )
+            .file("src/lib.rs", "")
+            .build();
+        p.cargo("fetch")
+            .env("CARGO_LOG", "cargo::sources::git::utils=debug")
+            .with_stderr_contains("[UPDATING] git repository `https://github.com/rust-lang/bitflags.git`")
+            .with_stderr_contains("[..]attempting GitHub fast path[..]")
+            .with_stderr_contains(&format!("[..]github fast path fetch {oid}"))
+            .with_stderr_contains(&format!("[..]initiating fetch of [{refspecs}] from https://github.com/rust-lang/bitflags.git"))
+            .run();
+        // Remove the lock file, and test the up-to-date code path.
+        p.root().join("Cargo.lock").rm_rf();
+        p.cargo("fetch")
+            .env("CARGO_LOG", "cargo::sources::git::utils=debug")
+            .with_stderr_contains(&format!("[..]{up_to_date}"))
+            .run();
+
+        paths::home().join(".cargo/git").rm_rf();
+    }
 }