Add support for Git dependencies (#283)

## Summary

This PR adds support for Git dependencies, like:

```
flask @ git+https://github.com/pallets/flask.git
```

Right now, they're only supported in the resolver (and not the
installer), since the installer doesn't yet support source distributions
at all.

The general approach here is based on Cargo's Git implementation.
Specifically, I adapted Cargo's
[`git`](https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/sources/git/mod.rs)
module to perform the cloning, which is based on `libgit2`.

As compared to Cargo's implementation, I made the following changes:

- Removed any unnecessary code.
- Fixed any Clippy errors for our stricter ruleset.
- Removed the dependency on `curl`, in favor of `reqwest` which we use
elsewhere.
- Removed the ability to use `gix`. Cargo allows the use of `gix` as an
experimental flag, but it only supports a small subset of the
operations. When Cargo fully adopts `gix`, we should plan to do the
same.
- Removed Cargo's host key checking. We need to re-add this! I'll do it
shortly.
- Removed Cargo's progress bars. We should re-add this too, but we use
`indicatif` and Cargo had their own thing.

There are a few follow-ups to consider:

- Adding support in the installer.
- When we lock, we should write out the Git URL that includes the exact
SHA. This lets us cache in perpetuity and avoids dependencies changing
without re-locking.
- When we resolve, we should _always_ try to refresh Git dependencies.
(Right now, we skip if the wheel was already built.)

I'll work on the latter two in follow-up PRs.

Closes #202.

Cargo.toml

@@ -29,6 +29,7 @@ fs-err = { version = "2.9.0" }
 fs2 = { version = "0.4.3" }
 futures = { version = "0.3.28" }
 fxhash = { version = "0.2.1" }
+glob = { version = "0.3.1" }
 goblin = { version = "0.7.1" }
 hex = { version = "0.4.3" }
 http-cache-reqwest = { version = "0.11.3" }
@@ -42,6 +43,7 @@ petgraph = { version = "0.6.4" }
 platform-info = { version = "2.0.2" }
 plist = { version = "1.5.0" }
 pyproject-toml = { version = "0.7.0" }
+rand = { version = "0.8.5" }
 rayon = { version = "1.8.0" }
 reflink-copy = { version = "0.1.10" }
 regex = { version = "1.9.6" }

crates/puffin-build/src/lib.rs

@@ -117,7 +117,8 @@ pub struct SourceDistributionBuilder {
 }
 
 impl SourceDistributionBuilder {
-    /// Extract the source distribution and create a venv with the required packages
+    /// Create a virtual environment in which to build a source distribution, extracting the
+    /// contents from an archive if necessary.
     pub async fn setup(
         sdist: &Path,
         interpreter_info: &InterpreterInfo,
@@ -126,9 +127,13 @@ impl SourceDistributionBuilder {
         let temp_dir = tempdir()?;
 
         // TODO(konstin): Parse and verify filenames
-        debug!("Unpacking for build {}", sdist.display());
-        let extracted = temp_dir.path().join("extracted");
-        let source_tree = extract_archive(sdist, &extracted)?;
+        let source_tree = if fs::metadata(sdist)?.is_dir() {
+            sdist.to_path_buf()
+        } else {
+            debug!("Unpacking for build: {}", sdist.display());
+            let extracted = temp_dir.path().join("extracted");
+            extract_archive(sdist, &extracted)?
+        };
 
         // Check if we have a PEP 517 build, a legacy setup.py, or an edge case
         let build_system = if source_tree.join("pyproject.toml").is_file() {

crates/puffin-cli/tests/pip_compile.rs

@@ -627,6 +627,46 @@ fn compile_sdist_url_dependency() -> Result<()> {
     Ok(())
 }
 
+/// Resolve a specific Flask source distribution via a Git HTTPS dependency.
+#[test]
+fn compile_git_https_dependency() -> Result<()> {
+    let temp_dir = assert_fs::TempDir::new()?;
+    let cache_dir = assert_fs::TempDir::new()?;
+    let venv = temp_dir.child(".venv");
+
+    Command::new(get_cargo_bin(BIN_NAME))
+        .arg("venv")
+        .arg(venv.as_os_str())
+        .arg("--cache-dir")
+        .arg(cache_dir.path())
+        .current_dir(&temp_dir)
+        .assert()
+        .success();
+    venv.assert(predicates::path::is_dir());
+
+    let requirements_in = temp_dir.child("requirements.in");
+    requirements_in.touch()?;
+    requirements_in.write_str("flask @ git+https://github.com/pallets/flask.git")?;
+
+    insta::with_settings!({
+        filters => vec![
+            (r"(\d|\.)+(ms|s)", "[TIME]"),
+            (r"#    .* pip-compile", "#    [BIN_PATH] pip-compile"),
+            (r"--cache-dir .*", "--cache-dir [CACHE_DIR]"),
+        ]
+    }, {
+        assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+            .arg("pip-compile")
+            .arg("requirements.in")
+            .arg("--cache-dir")
+            .arg(cache_dir.path())
+            .env("VIRTUAL_ENV", venv.as_os_str())
+            .current_dir(&temp_dir));
+    });
+
+    Ok(())
+}
+
 /// Request Flask, but include a URL dependency for Werkzeug, which should avoid adding a
 /// duplicate dependency from `PyPI`.
 #[test]

crates/puffin-cli/tests/snapshots/pip_compile__compile_git_https_dependency.snap

@@ -0,0 +1,36 @@
+---
+source: crates/puffin-cli/tests/pip_compile.rs
+info:
+  program: puffin
+  args:
+    - pip-compile
+    - requirements.in
+    - "--cache-dir"
+    - /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpbvYz3u
+  env:
+    VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpHYkK5F/.venv
+---
+success: true
+exit_code: 0
+----- stdout -----
+# This file was autogenerated by Puffin v0.0.1 via the following command:
+#    [BIN_PATH] pip-compile requirements.in --cache-dir [CACHE_DIR]
+blinker==1.7.0
+    # via flask
+click==8.1.7
+    # via flask
+flask @ git+https://github.com/pallets/flask.git
+itsdangerous==2.1.2
+    # via flask
+jinja2==3.1.2
+    # via flask
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
+werkzeug==3.0.1
+    # via flask
+
+----- stderr -----
+Resolved 7 packages in [TIME]
+