feat: Implement wnfs-unixfs-file crate for encoding big byte arrays in IPLD

Cargo.toml

@@ -5,6 +5,7 @@ members = [
 	"wnfs-common",
 	"wnfs-hamt",
 	"wnfs-nameaccumulator",
+	"wnfs-unixfs-file",
 	"wnfs-wasm",
 ]
 

README.md

@@ -53,10 +53,27 @@ This library is designed with WebAssembly in mind. You can follow instructions o
 ## Crates
 
 - [wnfs](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs)
+- [wnfs-wasm](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-wasm)
 - [wnfs-common](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-common)
 - [wnfs-hamt](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-hamt)
 - [wnfs-nameaccumulator](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-nameaccumulator)
-- [wnfs-wasm](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-wasm)
+- [wnfs-unixfs-file](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-unixfs-file)
+
+This is the dependency graph between these crates:
+```mermaid
+flowchart TD
+    wnfs-wasm --> wnfs
+    wnfs-wasm --> wnfs-nameaccumulator
+    %% wnfs-bench --> wnfs
+    %% wnfs-bench --> wnfs-hamt
+    %% wnfs-bench --> wnfs-nameaccumulator
+    wnfs --> wnfs-hamt
+    wnfs --> wnfs-common
+    wnfs --> wnfs-unixfs-file
+    wnfs --> wnfs-nameaccumulator
+    wnfs-unixfs-file --> wnfs-common
+    wnfs-hamt --> wnfs-common
+```
 
 ## Building the Project
 

wnfs-hamt/src/strategies/changes.rs

@@ -34,7 +34,7 @@ pub(crate) fn generate_changes<K: Debug + Clone, V: Debug + Clone>(
         pairs
             .clone()
             .into_iter()
-            .zip(randoms.into_iter())
+            .zip(randoms)
             .filter(|(_, (num, _))| *num != 0)
             .map(|((k, _), (num, val))| match num {
                 1 => Change::Add(k, val),

wnfs-unixfs-file/CHANGELOG.md

@@ -0,0 +1 @@
+# Changelog

wnfs-unixfs-file/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "wnfs-unixfs-file"
+version = "0.1.25"
+description = "IPLD UnixFS File implementation for Webnative Filesystem"
+keywords = ["wnfs", "unixfs", "webnative", "ipfs", "decentralisation"]
+categories = [
+  "filesystem",
+  "cryptography",
+  "web-programming",
+  "wasm",
+]
+license = "Apache-2.0"
+readme = "README.md"
+edition = "2021"
+repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-unixfs-file"
+homepage = "https://fission.codes"
+authors = ["The Fission Authors"]
+
+[dependencies]
+anyhow = "1.0"
+async-stream = "0.3"
+bytes = "1.5"
+futures = "0.3"
+libipld = { version = "0.16", features = [] }
+num_enum = "0.5"
+proptest = { version = "1.1", optional = true }
+prost = "0.12"
+rand_core = "0.6"
+testresult = "0.3"
+tokio = { version = "1.34", features = ["io-util"] }
+wnfs-common = { path = "../wnfs-common", version = "=0.1.25" }
+
+[dev-dependencies]
+async-std = { version = "1.11", features = ["attributes"] }
+data-encoding = "2.5.0"
+proptest = "1.1"
+rand = "0.8"
+rand_chacha = "0.3.1"
+tempfile = "3.8.1"
+test-strategy = "0.3"
+tokio = { version = "1.34", features = ["fs", "rt", "macros"] }
+tokio-test = "0.4.3"
+tokio-util = { version = "0.7", features = ["io"] }
+wnfs-common = { path = "../wnfs-common", features = ["test_utils"] }

wnfs-unixfs-file/README.md

@@ -0,0 +1,79 @@
+<div align="center">
+  <a href="https://github.com/wnfs-wg" target="_blank">
+    <img src="https://raw.githubusercontent.com/wnfs-wg/rs-wnfs/main/assets/logo.png" alt="WNFS Logo" width="100" height="100"></img>
+  </a>
+
+  <h1 align="center">wnfs-unixfs-file</h1>
+
+  <p>
+    <a href="https://crates.io/crates/wnfs-unixfs-file">
+      <img src="https://img.shields.io/crates/v/wnfs-unixfs-file?label=crates" alt="Docs">
+    </a>
+    <a href="https://codecov.io/gh/wnfs-wg/rs-wnfs">
+      <img src="https://codecov.io/gh/wnfs-wg/rs-wnfs/branch/main/graph/badge.svg?token=95YHXFMFF4" alt="Code Coverage"/>
+    </a>
+    <a href="https://github.com/wnfs-wg/rs-wnfs/actions?query=">
+      <img src="https://github.com/wnfs-wg/rs-wnfs/actions/workflows/checks.yaml/badge.svg" alt="Build Status">
+    </a>
+    <a href="https://github.com/wnfs-wg/rs-wnfs/blob/main/LICENSE">
+      <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License">
+    </a>
+    <a href="https://docs.rs/wnfs">
+      <img src="https://img.shields.io/static/v1?label=Docs&message=docs.rs&color=blue" alt="Docs">
+    </a>
+    <a href="https://discord.gg/zAQBDEq">
+      <img src="https://img.shields.io/static/v1?label=Discord&message=join%20us!&color=mediumslateblue" alt="Discord">
+    </a>
+  </p>
+</div>
+
+<div align="center"><sub>:warning: Work in progress :warning:</sub></div>
+
+##
+
+This Rust crate provides an implementation of UnixFs files. WNFS uses the UnixFs file encoding purely to chunk big byte arrays into multiple blocks and produce a single CID for them to link to from WNFS structures.
+
+This crate is a fork from beetle (previously "iroh")'s [iroh-unixfs crate](https://github.com/n0-computer/beetle/tree/3e137cb2bc18e1d458c3f72d5e817b03d9537d5d/iroh-unixfs).
+
+Major changes relative to that implementation include:
+- Removed prost code generation, instead it's some hard-coded structs with prost annotations
+- Removed support for any UnixFs structures other than files (no directories, directory shards or symlinks)
+- Removed parallelization for hashing to make the crate async runtime-independent (so it can be used in wasm with wasm-bindgen-futures!)
+- Doesn't hard-code use of SHA-256 anymore
+- Integrated with the wnfs-common `BlockStore` trait
+
+## Usage
+
+```rs
+use wnfs_unixfs_file::builder::FileBuilder;
+use wnfs_common::MemoryBlockStore;
+use tokio::io::AsyncReadExt;
+
+// Where data is stored
+let store = &MemoryBlockStore::new();
+
+// Encoding byte arrays, getting a CID
+let data = vec![1u8; 1_000_000]; // 1MiB of ones
+let root_cid = FileBuilder::new()
+    .content_bytes(data)
+    .build()?
+    .store(store)
+    .await?;
+
+// Taking a CID, reading back a byte array:
+let file = UnixFsFile::load(&root_cid, store).await?;
+println!("filesize: {}", file.filesize());
+let mut buffer = Vec::new();
+let mut reader = file.into_content_reader(store, None)?;
+reader.read_to_end(&mut buffer).await?;
+// buffer now has 1 million ones
+
+// You can also seek
+use tokio::io::AsyncSeekExt;
+use std::io::SeekFrom;
+let mut reader = file.into_content_reader(store, None)?;
+reader.seek(SeekFrom::Start(10_000)).await?;
+let mut slice = [0u8; 10_000];
+reader.read_exact(&mut slice).await?;
+// slice now has 10_000 ones
+```

wnfs-unixfs-file/proptest-regressions/builder.txt

@@ -0,0 +1,8 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 376480a4a772236f8ba598f595ea690a0a12ff4456b4fe61a9d8caccc03b9a17 # shrinks to input = _TestEncodeDecodeRoundtripArgs { data: [] }
+cc 77c264b383e78eb742f9da87cbff6aa4e6de7d87991ef955792e5637b8134e46 # shrinks to input = _TestSeekSubarrayArgs { seed: 0, degree: 2, len: 87383, seek_start: 43680, seek_len: 23, chunker: Rabin }