feat: add HAMT sharded directories support (#536)

packages/upload-client/package.json

@@ -65,7 +65,7 @@
   "dependencies": {
     "@ipld/car": "^5.0.3",
     "@ipld/dag-ucan": "^3.2.0",
-    "@ipld/unixfs": "^2.0.1",
+    "@ipld/unixfs": "^2.1.1",
     "@ucanto/client": "^5.1.0",
     "@ucanto/interface": "^6.0.0",
     "@ucanto/transport": "^5.1.0",

packages/upload-client/src/unixfs.js

@@ -1,12 +1,16 @@
 import * as UnixFS from '@ipld/unixfs'
 import * as raw from 'multiformats/codecs/raw'
+import { withMaxChunkSize } from '@ipld/unixfs/file/chunker/fixed'
+import { withWidth } from '@ipld/unixfs/file/layout/balanced'
 
+const SHARD_THRESHOLD = 1000 // shard directory after > 1,000 items
 const queuingStrategy = UnixFS.withCapacity()
 
-// TODO: configure chunk size and max children https://github.com/ipld/js-unixfs/issues/36
 const settings = UnixFS.configure({
   fileChunkEncoder: raw,
   smallFileEncoder: raw,
+  chunker: withMaxChunkSize(1024 * 1024),
+  fileLayout: withWidth(1024),
 })
 
 /**
@@ -64,7 +68,10 @@ class UnixFSDirectoryBuilder {
 
   /** @param {import('@ipld/unixfs').View} writer */
   async finalize(writer) {
-    const dirWriter = UnixFS.createDirectoryWriter(writer)
+    const dirWriter =
+      this.entries.size <= SHARD_THRESHOLD
+        ? UnixFS.createDirectoryWriter(writer)
+        : UnixFS.createShardedDirectoryWriter(writer)
     for (const [name, entry] of this.entries) {
       const link = await entry.finalize(writer)
       dirWriter.set(name, link)

packages/upload-client/test/index.test.js

@@ -113,7 +113,7 @@ describe('uploadFile', () => {
   it('allows custom shard size to be set', async () => {
     const space = await Signer.generate()
     const agent = await Signer.generate() // The "user" that will ask the service to accept the upload
-    const file = new Blob([await randomBytes(500_000)])
+    const file = new Blob([await randomBytes(1024 * 1024 * 5)])
     /** @type {import('../src/types').CARLink[]} */
     const carCIDs = []
 
@@ -174,12 +174,12 @@ describe('uploadFile', () => {
       file,
       {
         connection,
-        shardSize: 400_000, // should end up with 2 CAR files
+        shardSize: 1024 * 1024 * 2, // should end up with 2 CAR files
         onShardStored: (meta) => carCIDs.push(meta.cid),
       }
     )
 
-    assert.equal(carCIDs.length, 2)
+    assert.equal(carCIDs.length, 3)
   })
 })
 

packages/upload-client/test/sharding.test.js

@@ -15,8 +15,8 @@ import { mockService } from './helpers/mocks.js'
 
 describe('ShardingStream', () => {
   it('creates shards from blocks', async () => {
-    const file = new Blob([await randomBytes(1024 * 1024)])
-    const shardSize = 512 * 1024
+    const file = new Blob([await randomBytes(1024 * 1024 * 5)])
+    const shardSize = 1024 * 1024 * 2
 
     /** @type {import('../src/types').CARFile[]} */
     const shards = []

packages/upload-client/test/unixfs.test.js

@@ -1,4 +1,5 @@
 import assert from 'assert'
+import { decode, NodeType } from '@ipld/unixfs'
 import { exporter } from 'ipfs-unixfs-exporter'
 import { MemoryBlockstore } from 'blockstore-core/memory'
 import * as raw from 'multiformats/codecs/raw'
@@ -67,6 +68,31 @@ describe('UnixFS', () => {
     expectedPaths.forEach((p) => assert(actualPaths.includes(p)))
   })
 
+  it('encodes a sharded directory', async () => {
+    const files = []
+    for (let i = 0; i < 1001; i++) {
+      files.push(new File([`data${i}`], `file${i}.txt`))
+    }
+
+    const { cid, blocks } = await encodeDirectory(files)
+    const blockstore = await blocksToBlockstore(blocks)
+    const dirEntry = await exporter(cid.toString(), blockstore)
+    assert.equal(dirEntry.type, 'directory')
+
+    const expectedPaths = files.map((f) => path.join(cid.toString(), f.name))
+    // @ts-expect-error
+    const entries = await collectDir(dirEntry)
+    const actualPaths = entries.map((e) => e.path)
+
+    expectedPaths.forEach((p) => assert(actualPaths.includes(p)))
+
+    // check root node is a HAMT sharded directory
+    // @ts-expect-error
+    const bytes = await blockstore.get(cid)
+    const node = decode(bytes)
+    assert.equal(node.type, NodeType.HAMTShard)
+  })
+
   it('throws then treating a file as a directory', () =>
     assert.rejects(
       encodeDirectory([