Skip to content

Commit

Permalink
feat(api): rewrite entire API to be like std::fs
Browse files Browse the repository at this point in the history 
Fixes: #13

BREAKING CHANGE: The entire API surface has been rewritten and overhauled. Instead of breaking up the package into get/put/rm/ls modules, everything is now available mostly through the toplevel module, following the same conventions as std:fs for its operations. Associated structs and other types have also been renamed. Please refer to the documentation for the new API, and I hope you find it more ergonomic!
  • Loading branch information
@zkat
zkat committed Oct 22, 2019
1 parent ccdd8c7 commit 773296a
Show file tree
Hide file tree
Showing 8 changed files with 630 additions and 418 deletions.
9 changes: 6 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ async fn main() -> Result<(), cacache::Error> {
let dir = String::from("./my-cache");

// Write some data!
cacache::put::data(&dir, "key", b"my-async-data").await?;
cacache::write(&dir, "key", b"my-async-data").await?;

// Get the data back!
let data = cacache::get::data(&dir, "key").await?;
let data = cacache::read(&dir, "key").await?;
assert_eq!(data, b"my-async-data");

// Clean up the data!
Expand All @@ -36,18 +36,21 @@ Using [`cargo-edit`](https://crates.io/crates/cargo-edit)

## Features

- First-class async support, using [`async-std`](https://crates.io/crates/async-std) as its runtime. Sync APIs are available but secondary.
- First-class async support, using [`async-std`](https://crates.io/crates/async-std) as its runtime. Sync APIs are available but secondary
- `std::fs`-style API
- Extraction by key or by content address (shasum, etc)
- [Subresource Integrity](#integrity) web standard support
- Multi-hash support - safely host sha1, sha512, etc, in a single cache
- Automatic content deduplication
- Atomic content writes even for large data
- Fault tolerance (immune to corruption, partial writes, process races, etc)
- Consistency guarantees on read and write (full data verification)
- Lockless, high-concurrency cache access
- Really helpful, contextual error messages
- Large file support
- Pretty darn fast
- Arbitrary metadata storage
- Cross-platform: Windows and case-(in)sensitive filesystem support
- Punches nazis

## Contributing
Expand Down
73 changes: 32 additions & 41 deletions benches/benchmarks.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,17 +73,17 @@ fn baseline_read_many_async(c: &mut Criterion) {
});
}

fn get_data_hash_sync(c: &mut Criterion) {
fn read_hash_sync(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data = b"hello world".to_vec();
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap();
let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data_hash_sync", move |b| {
b.iter(|| cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap())
b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap())
});
}

fn get_data_hash_many_sync(c: &mut Criterion) {
fn read_hash_many_sync(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data: Vec<_> = (0..)
Expand All @@ -92,41 +92,38 @@ fn get_data_hash_many_sync(c: &mut Criterion) {
.collect();
let sris: Vec<_> = data
.iter()
.map(|datum| cacache::put::data_sync(&cache, "hello", datum).unwrap())
.map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap())
.collect();
c.bench_function("get::data_hash_many_sync", move |b| {
b.iter(|| {
for sri in sris.iter() {
cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap();
cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap();
}
})
});
}

fn get_data_sync(c: &mut Criterion) {
fn read_sync(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data = b"hello world".to_vec();
cacache::put::data_sync(&cache, "hello", data).unwrap();
cacache::get::data_sync(&cache, "hello").unwrap();
cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data_sync", move |b| {
b.iter(|| {
cacache::get::data_sync(black_box(&cache), black_box(String::from("hello"))).unwrap()
})
b.iter(|| cacache::read_sync(black_box(&cache), black_box(String::from("hello"))).unwrap())
});
}

fn get_data_hash_sync_big_data(c: &mut Criterion) {
fn read_hash_sync_big_data(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data = vec![1; 1024 * 1024 * 5];
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap();
let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get_hash_big_data", move |b| {
b.iter(|| cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap())
b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap())
});
}

fn get_data_hash_many_async(c: &mut Criterion) {
fn read_hash_many_async(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data: Vec<_> = (0..)
Expand All @@ -135,51 +132,45 @@ fn get_data_hash_many_async(c: &mut Criterion) {
.collect();
let sris: Vec<_> = data
.iter()
.map(|datum| cacache::put::data_sync(&cache, "hello", datum).unwrap())
.map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap())
.collect();
c.bench_function("get::data_hash_many", move |b| {
b.iter(|| {
let tasks = sris
.iter()
.map(|sri| cacache::get::data_hash(black_box(&cache), black_box(&sri)));
.map(|sri| cacache::read_hash(black_box(&cache), black_box(&sri)));
task::block_on(futures::future::join_all(tasks));
})
});
}

fn get_data_hash_async(c: &mut Criterion) {
fn read_hash_async(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data = b"hello world".to_vec();
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap();
let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data_hash", move |b| {
b.iter(|| {
task::block_on(cacache::get::data_hash(black_box(&cache), black_box(&sri))).unwrap()
})
b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
});
}

fn get_data_async(c: &mut Criterion) {
fn read_async(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data = b"hello world".to_vec();
cacache::put::data_sync(&cache, "hello", data).unwrap();
cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data", move |b| {
b.iter(|| {
task::block_on(cacache::get::data(black_box(&cache), black_box("hello"))).unwrap()
})
b.iter(|| task::block_on(cacache::read(black_box(&cache), black_box("hello"))).unwrap())
});
}

fn get_data_hash_async_big_data(c: &mut Criterion) {
fn read_hash_async_big_data(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned();
let data = vec![1; 1024 * 1024 * 5];
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap();
let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data_big_data", move |b| {
b.iter(|| {
task::block_on(cacache::get::data_hash(black_box(&cache), black_box(&sri))).unwrap()
})
b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
});
}

Expand All @@ -189,13 +180,13 @@ criterion_group!(
baseline_read_many_sync,
baseline_read_async,
baseline_read_many_async,
get_data_hash_async,
get_data_hash_many_async,
get_data_hash_sync,
get_data_hash_many_sync,
get_data_async,
get_data_sync,
get_data_hash_async_big_data,
get_data_hash_sync_big_data
read_hash_async,
read_hash_many_async,
read_hash_sync,
read_hash_many_sync,
read_async,
read_sync,
read_hash_async_big_data,
read_hash_sync_big_data
);
criterion_main!(benches);
Loading

0 comments on commit 773296a

Please sign in to comment.