fix: trickle dags

ipld · Mar 10, 2022 · 6f0b501 · 6f0b501
1 parent 19f059e
commit 6f0b501
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 6 deletions.
diff --git a/src/file.js b/src/file.js
@@ -40,6 +40,12 @@ export const UnixFSLeaf = {
   encode: UnixFS.encodeFileChunk,
 }
 
+export const UnixFSRawLeaf = {
+  code: UnixFS.code,
+  name: UnixFS.name,
+  encode: UnixFS.encodeRaw,
+}
+
 /**
  * @template [Layout=unknown]
  * @param {UnixFS.Metadata} [metadata]

diff --git a/src/file/layout/trickle.js b/src/file/layout/trickle.js
@@ -58,7 +58,7 @@ export const open = options => ({
   leafCount: 0,
   levelCutoffs: [],
   tail: null,
-  lastID: EMTPY_LEAF_ID + 1,
+  lastID: EMTPY_LEAF_ID,
 })
 
 /**

diff --git a/src/file/writer.js b/src/file/writer.js
@@ -135,6 +135,7 @@ export const write = (state, bytes) => {
       state.layout,
       chunks
     )
+
     const { linked, ...nodeQueue } = Queue.addNodes(nodes, state.nodeQueue)
 
     // Create leaf encode tasks for all new leaves
@@ -230,6 +231,7 @@ export const close = state => {
         layout,
         state.metadata
       )
+
       const { linked, ...nodeQueue } = Queue.addNodes(
         [...nodes, ...rest, root],
         state.nodeQueue

diff --git a/test/file.spec.js b/test/file.spec.js
@@ -163,8 +163,9 @@ describe("test file", () => {
     const { writer, ...importer } = FileImporter.createImporter(
       {},
       FileImporter.configure({
-        chunker: FixedSize.withMaxChunkSize(CHUNK_SIZE),
+        chunker: FixedSize.withMaxChunkSize(1300),
         fileLayout: Trickle,
+        fileChunkEncoder: FileImporter.UnixFSRawLeaf,
       })
     )
     const collector = collect(importer.blocks)
@@ -177,10 +178,41 @@ describe("test file", () => {
 
     assert.deepEqual(link, {
       cid: CID.parse(
-        "bafybeifovo36yvep6m53p3ghmrx6a3ig6c2ydvxp4yjmlqgg74clvudesy"
+        "bafybeidia54tfr7ycw2ls2mxyjpcto42mriytx2ymlwgwsjqzner5wqc5u"
       ),
       contentByteLength: 524288,
-      dagByteLength: 524424,
+      dagByteLength: 548251,
+    })
+  })
+
+  it("trickle layout with overflow", async function () {
+    this.timeout(30000)
+    const content = hashrecur({
+      byteLength: CHUNK_SIZE * 2,
+    })
+
+    const { writer, ...importer } = FileImporter.createImporter(
+      {},
+      FileImporter.configure({
+        chunker: FixedSize.withMaxChunkSize(100000),
+        fileLayout: Trickle.configure({ maxDirectLeaves: 5 }),
+        fileChunkEncoder: FileImporter.UnixFSRawLeaf,
+      })
+    )
+    const collector = collect(importer.blocks)
+
+    for await (const slice of content) {
+      writer.write(slice)
+    }
+    const link = await writer.close()
+    const blocks = await collector
+
+    assert.deepEqual(link, {
+      cid: CID.parse(
+        "bafybeigu6bkvpxtamauopeu2ejzkxy4wgqa576wmfc6ubusjwhgold4aum"
+      ),
+      contentByteLength: 524288,
+      dagByteLength: 524738,
     })
   })
 })
diff --git a/test/matrix.js b/test/matrix.js
@@ -4,7 +4,7 @@ import * as Balanced from "../src/file/layout/balanced.js"
 import * as FixedSize from "../src/file/chunker/fixed.js"
 import * as Rabin from "../src/file/chunker/rabin.new.js"
 import * as API from "../src/file/api.js"
-import { UnixFSLeaf } from "../src/file.js"
+import { UnixFSLeaf, UnixFSRawLeaf } from "../src/file.js"
 import * as RawLeaf from "multiformats/codecs/raw"
 import * as UnixFS from "../src/unixfs.js"
 import { sha256 } from "multiformats/hashes/sha2"
@@ -47,7 +47,11 @@ const base = new URL("./dataset/testdata/", import.meta.url)
  */
 export const parseConfig = async input => {
   const url = new URL(input.source, base)
-  const fileChunkEncoder = input.rawLeaves ? RawLeaf : UnixFSLeaf
+  const fileChunkEncoder = input.rawLeaves
+    ? RawLeaf
+    : input.trickle
+    ? UnixFSRawLeaf
+    : UnixFSLeaf
 
   return {
     url: url,