feature(async): Add tokio as an executor option (#36)

This PR is based on the work @alexschrod did in PR #29. All
I did was carry it over the finish line.

This PR adds a feature to the crate named `tokio-runtime`. If
you disable default features and enable this new one, cacache
uses tokio as its async executor. This makes integrating cacache
with tokio-using projects easier, because the file types leak out
if you use anything more than the top-level convenience functions.

The PR implements the feature using shims in a new submodule named
`async_lib`. This module conditionally uses either async-std
or tokio based on feature selection, and hides some differences with
convenience functions.

This change should not be a breaking change, because the default is
still async-std.

There are a few other small changes in this PR worth noting.

- The README shows how to switch runtimes.
- There's a justfile to run common tasks, including those in makefile.toml.
  The default shell is `sh`, so this might not work out of the box for
  Windows users.
- The tests can now run under either runtime. The justfile has a recipe
  that runs them both.
- The benchmarks can also run under either runtime. The justfile has two
  recipe for this, one using bench and the other using criterion's runner.
- The dependencies now pull in async-attributes by default along with
  async-std. This made it easier to swap runtimes in the tests.
- All dependency versions have been bumped.

Co-authored-by: @alexschrod

Cargo.toml

@@ -8,32 +8,41 @@ license = "Apache-2.0"
 repository = "https://github.com/zkat/cacache-rs"
 homepage = "https://github.com/zkat/cacache-rs"
 readme = "README.md"
-categories = [
- "caching",
- "filesystem"
-]
+categories = ["caching", "filesystem"]
 
 [dependencies]
-ssri = "7.0.0"
+async-attributes = { version = "1.1.2", optional = true }
+async-std = { version = "1.10.0", features = ["unstable"], optional = true }
+digest = "0.10.6"
+either = "1.6.1"
+futures = "0.3.17"
 hex = "0.4.3"
-tempfile = "3.2.0"
-sha-1 = "0.9.8"
-sha2 = "0.9.8"
-digest = "0.9.0"
-serde_json = "1.0.68"
+memmap2 = "0.5.8"
 serde = "1.0.130"
 serde_derive = "1.0.130"
-walkdir = "2.3.2"
-either = "1.6.1"
-async-std = { version = "1.10.0", features = ["unstable"] }
+serde_json = "1.0.68"
+sha1 = "0.10.5"
+sha2 = "0.10.6"
+ssri = "7.0.0"
+tempfile = "3.2.0"
 thiserror = "1.0.29"
-futures = "0.3.17"
-memmap2 = "0.5"
+tokio = { version = "1.12.0", features = [
+ "fs",
+ "io-util",
+ "macros",
+ "rt",
+ "rt-multi-thread",
+], optional = true }
+tokio-stream = { version = "0.1.7", features = ["io-util"], optional = true }
+walkdir = "2.3.2"
 
 [dev-dependencies]
-async-attributes = "1.1.2"
-criterion = "0.3.5"
+criterion = "0.4.0"
 
 [[bench]]
 name = "benchmarks"
 harness = false
+
+[features]
+default = ["async-std", "async-attributes"]
+tokio-runtime = ["tokio", "tokio-stream"]

README.md

@@ -38,7 +38,7 @@ Minimum supported Rust version is `1.43.0`.
 
 ## Features
 
-- First-class async support, using [`async-std`](https://crates.io/crates/async-std) as its runtime. Sync APIs are available but secondary
+- First-class async support, using either [`async-std`](https://crates.io/crates/async-std) or [`tokio`](https://crates.io/crates/tokio) as its runtime. Sync APIs are available but secondary
 - `std::fs`-style API
 - Extraction by key or by content address (shasum, etc)
 - [Subresource Integrity](#integrity) web standard support
@@ -55,6 +55,13 @@ Minimum supported Rust version is `1.43.0`.
 - Cross-platform: Windows and case-(in)sensitive filesystem support
 - Punches nazis
 
+`async-std` is the default async runtime. To use `tokio` instead, turn off default features and enable the `tokio-runtime` feature, like this:
+
+```toml
+[dependencies]
+cacache = { version = "*", default-features = false, features = ["tokio-runtime"] }
+```
+
 ## Contributing
 
 The cacache team enthusiastically welcomes contributions and project participation! There's a bunch of things you can do if you want to contribute! The [Contributor Guide](CONTRIBUTING.md) has all the information you need for everything from reporting bugs to contributing entire new features. Please don't hesitate to jump in if you'd like to, or even ask us questions if something isn't clear.

benches/benchmarks.rs

@@ -1,4 +1,19 @@
-use async_std::{fs as afs, task};
+#[cfg(feature = "async-std")]
+use async_std::fs as afs;
+#[cfg(all(test, feature = "tokio"))]
+use tokio::fs as afs;
+
+#[cfg(all(test, feature = "async-std"))]
+pub use async_std::task::block_on;
+#[cfg(all(test, feature = "tokio"))]
+#[inline]
+pub fn block_on<F, T>(future: F) -> T
+where
+ F: std::future::Future<Output = T>,
+{
+ tokio::runtime::Runtime::new().unwrap().block_on(future)
+}
+
 use std::fs::{self, File};
 use std::io::prelude::*;
 
@@ -47,7 +62,7 @@ fn baseline_read_async(c: &mut Criterion) {
  fd.write_all(data).unwrap();
  drop(fd);
  c.bench_function("baseline_read_async", move |b| {
- b.iter(|| task::block_on(afs::read(&path)))
+ b.iter(|| block_on(afs::read(&path)))
  });
 }
 
@@ -66,7 +81,7 @@ fn baseline_read_many_async(c: &mut Criterion) {
  c.bench_function("baseline_read_many_async", move |b| {
  b.iter(|| {
  let tasks = paths.iter().map(|path| afs::read(black_box(path)));
- task::block_on(futures::future::join_all(tasks));
+ block_on(futures::future::join_all(tasks));
  })
  });
 }
@@ -137,7 +152,7 @@ fn read_hash_many_async(c: &mut Criterion) {
  let tasks = sris
  .iter()
  .map(|sri| cacache::read_hash(black_box(&cache), black_box(sri)));
- task::block_on(futures::future::join_all(tasks));
+ block_on(futures::future::join_all(tasks));
  })
  });
 }
@@ -148,7 +163,7 @@ fn read_hash_async(c: &mut Criterion) {
  let data = b"hello world".to_vec();
  let sri = cacache::write_sync(&cache, "hello", data).unwrap();
  c.bench_function("get::data_hash", move |b| {
- b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
+ b.iter(|| block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
  });
 }
 
@@ -158,7 +173,7 @@ fn read_async(c: &mut Criterion) {
  let data = b"hello world".to_vec();
  cacache::write_sync(&cache, "hello", data).unwrap();
  c.bench_function("get::data", move |b| {
- b.iter(|| task::block_on(cacache::read(black_box(&cache), black_box("hello"))).unwrap())
+ b.iter(|| block_on(cacache::read(black_box(&cache), black_box("hello"))).unwrap())
  });
 }
 
@@ -168,7 +183,7 @@ fn read_hash_async_big_data(c: &mut Criterion) {
  let data = vec![1; 1024 * 1024 * 5];
  let sri = cacache::write_sync(&cache, "hello", data).unwrap();
  c.bench_function("get::data_big_data", move |b| {
- b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
+ b.iter(|| block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
  });
 }
 
@@ -179,7 +194,7 @@ fn write_hash_async(c: &mut Criterion) {
  b.iter_custom(|iters| {
  let start = std::time::Instant::now();
  for i in 0..iters {
- task::block_on(cacache::write_hash(&cache, format!("hello world{}", i))).unwrap();
+ block_on(cacache::write_hash(&cache, format!("hello world{}", i))).unwrap();
  }
  start.elapsed()
  })

justfile

@@ -0,0 +1,44 @@
+# List available just recipes
+@help:
+ just -l
+
+# Run tests on both runtimes with cargo nextest
+@test:
+ echo "----------\nasync-std:\n"
+ cargo nextest run
+ echo "\n----------\ntokio:\n"
+ cargo nextest run --no-default-features --features tokio-runtime
+
+# Run benchmarks with `cargo bench`
+@bench:
+ echo "----------\nasync-std:\n"
+ cargo bench
+ echo "\n----------\ntokio:\n"
+ cargo bench --no-default-features --features tokio-runtime
+
+# Run benchmarks with `cargo criterion`
+@criterion:
+ echo "----------\nasync-std:\n"
+ cargo criterion
+ echo "\n----------\ntokio:\n"
+ cargo criterion --no-default-features --features tokio-runtime
+
+# Generate a changelog with git-cliff
+changelog TAG:
+ git-cliff --prepend CHANGELOG.md -u --tag {{TAG}}
+
+# Prepare a release
+release *args:
+ cargo release --workspace {{args}}
+
+# Install workspace tools
+@install-tools:
+ cargo install cargo-nextest
+ cargo install cargo-release
+ cargo install git-cliff
+ cargo install cargo-criterion
+
+# Lint and automatically fix what we can fix
+@lint:
+ cargo clippy --fix --allow-dirty --allow-staged
+ cargo fmt

src/async_lib.rs

@@ -0,0 +1,125 @@
+#[cfg(feature = "async-std")]
+pub use async_std::fs::File;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::File;
+
+#[cfg(feature = "async-std")]
+pub use futures::io::AsyncRead;
+#[cfg(feature = "tokio")]
+pub use tokio::io::AsyncRead;
+
+#[cfg(feature = "async-std")]
+pub use futures::io::AsyncReadExt;
+#[cfg(feature = "tokio")]
+pub use tokio::io::AsyncReadExt;
+
+#[cfg(feature = "async-std")]
+pub use futures::io::AsyncBufReadExt;
+#[cfg(feature = "tokio")]
+pub use tokio::io::AsyncBufReadExt;
+
+#[cfg(feature = "async-std")]
+pub use futures::io::AsyncWrite;
+#[cfg(feature = "tokio")]
+pub use tokio::io::AsyncWrite;
+
+#[cfg(feature = "async-std")]
+pub use futures::io::AsyncWriteExt;
+#[cfg(feature = "tokio")]
+pub use tokio::io::AsyncWriteExt;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::read;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::read;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::copy;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::copy;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::metadata;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::metadata;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::remove_file;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::remove_file;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::create_dir_all;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::create_dir_all;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::remove_dir_all;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::remove_dir_all;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::DirBuilder;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::DirBuilder;
+
+#[cfg(feature = "async-std")]
+pub use async_std::fs::OpenOptions;
+#[cfg(feature = "tokio")]
+pub use tokio::fs::OpenOptions;
+
+#[cfg(feature = "async-std")]
+pub use async_std::io::BufReader;
+#[cfg(feature = "tokio")]
+pub use tokio::io::BufReader;
+
+#[cfg(feature = "async-std")]
+#[inline]
+pub fn lines_to_stream<R>(lines: futures::io::Lines<R>) -> futures::io::Lines<R> {
+ lines
+}
+#[cfg(feature = "tokio")]
+#[inline]
+pub fn lines_to_stream<R>(lines: tokio::io::Lines<R>) -> tokio_stream::wrappers::LinesStream<R> {
+ tokio_stream::wrappers::LinesStream::new(lines)
+}
+
+#[cfg(feature = "async-std")]
+pub use async_std::task::spawn_blocking;
+#[cfg(feature = "tokio")]
+pub use tokio::task::spawn_blocking;
+
+#[cfg(feature = "async-std")]
+pub use async_std::task::JoinHandle;
+#[cfg(feature = "async-std")]
+#[inline]
+pub fn unwrap_joinhandle_value<T>(value: T) -> T {
+ value
+}
+#[cfg(feature = "tokio")]
+pub use tokio::task::JoinHandle;
+#[cfg(feature = "tokio")]
+#[inline]
+pub fn unwrap_joinhandle_value<T>(value: Result<T, tokio::task::JoinError>) -> T {
+ value.unwrap()
+}
+
+use crate::errors::{Internal, InternalResult};
+use tempfile::NamedTempFile;
+
+#[cfg(feature = "async-std")]
+#[inline]
+pub async fn create_named_tempfile(tmp_path: std::path::PathBuf) -> InternalResult<NamedTempFile> {
+ spawn_blocking(|| NamedTempFile::new_in(tmp_path))
+ .await
+ .to_internal()
+}
+
+#[cfg(feature = "tokio")]
+#[inline]
+pub async fn create_named_tempfile(tmp_path: std::path::PathBuf) -> InternalResult<NamedTempFile> {
+ let tmpfile = spawn_blocking(|| NamedTempFile::new_in(tmp_path))
+ .await
+ .to_internal()?;
+ tmpfile.to_internal()
+}