feat: create a DAG from a list of CID’s and sizes

src/chunker/index.js

@@ -8,7 +8,12 @@ const chunkers = {
 }
 
 module.exports = (type, source, options) => {
-  const chunker = chunkers[type]
+  let chunker
+  if (typeof type === 'function') {
+    chunker = type
+  } else {
+    chunker = chunkers[type]
+  }
 
   if (!chunker) {
     throw errCode(new Error(`Unknkown chunker named ${type}`), 'ERR_UNKNOWN_CHUNKER')

src/dag-builder/file/index.js

@@ -27,23 +27,28 @@ async function * importBuffer (file, source, ipld, options) {
         ...options
       }
 
-      if (options.rawLeaves) {
-        node = buffer
-
-        opts.codec = 'raw'
-        opts.cidVersion = 1
+      let cid
+      if (options.fromParts) {
+        node = { length: buffer.size }
+        cid = buffer.cid
       } else {
-        unixfs = new UnixFS({
-          type: options.leafType,
-          data: buffer,
-          mtime: file.mtime,
-          mode: file.mode
-        })
-
-        node = new DAGNode(unixfs.marshal())
-      }
+        if (options.rawLeaves) {
+          node = buffer
 
-      const cid = await persist(node, ipld, opts)
+          opts.codec = 'raw'
+          opts.cidVersion = 1
+        } else {
+          unixfs = new UnixFS({
+            type: options.leafType,
+            data: buffer,
+            mtime: file.mtime,
+            mode: file.mode
+          })
+
+          node = new DAGNode(unixfs.marshal())
+        }
+        cid = await persist(node, ipld, opts)
+      }
 
       return {
         cid: cid,

src/dag-builder/index.js

@@ -30,7 +30,14 @@ async function * dagBuilder (source, ipld, options) {
         }
       }
 
-      const chunker = createChunker(options.chunker, validateChunks(source), options)
+      if (options.fromParts) {
+        options.rawLeaves = true
+        options.chunker = source => source
+        options.reduceSingleLeafToSelf = false
+      } else {
+        source = validateChunks(source)
+      }
+      const chunker = createChunker(options.chunker, source, options)
 
       // item is a file
       yield () => fileBuilder(entry, chunker, ipld, options)

test/chunker-custom.spec.js

@@ -0,0 +1,36 @@
+/* eslint-env mocha */
+'use strict'
+
+const importer = require('../src')
+
+const chai = require('chai')
+chai.use(require('dirty-chai'))
+const expect = chai.expect
+const IPLD = require('ipld')
+const inMemory = require('ipld-in-memory')
+
+// eslint bug https://github.com/eslint/eslint/issues/12459
+// eslint-disable-next-line require-await
+const iter = async function * () {
+  yield Buffer.from('one')
+  yield Buffer.from('two')
+}
+
+describe('custom chunker', function () {
+  it('keeps custom chunking', async () => {
+    const chunker = source => source
+    const content = iter()
+    const inmem = await inMemory(IPLD)
+    const sizes = [11, 11, 116]
+    const ipld = {
+      put: (node, ...args) => {
+        expect(node.toJSON().size).to.equal(sizes.shift())
+        return inmem.put(node, ...args)
+      }
+    }
+    for await (const part of importer([{ path: 'test', content }], ipld, { chunker })) {
+      expect(part.size).to.equal(116)
+    }
+    expect(sizes).to.be.empty()
+  })
+})

test/fromParts.spec.js

@@ -0,0 +1,43 @@
+/* eslint-env mocha */
+'use strict'
+
+const importer = require('../src')
+
+const chai = require('chai')
+chai.use(require('dirty-chai'))
+const expect = chai.expect
+const IPLD = require('ipld')
+const inMemory = require('ipld-in-memory')
+const CID = require('cids')
+
+const fromPartsTest = (iter, size) => async () => {
+  const content = iter()
+  const inmem = await inMemory(IPLD)
+  const sizes = [size]
+  const ipld = {
+    put: (node, ...args) => {
+      expect(node.toJSON().size).to.equal(sizes.shift())
+      return inmem.put(node, ...args)
+    }
+  }
+  for await (const part of importer([{ path: 'test', content }], ipld, { fromParts: true })) {
+    expect(part.size).to.equal(size)
+  }
+  expect(sizes).to.be.empty()
+}
+
+describe('custom chunker', function () {
+  // eslint bug https://github.com/eslint/eslint/issues/12459
+  // eslint-disable-next-line require-await
+  const multi = async function * () {
+    yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') }
+    yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') }
+  }
+  it('works with multiple parts', fromPartsTest(multi, 276312))
+
+  // eslint-disable-next-line require-await
+  const single = async function * () {
+    yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') }
+  }
+  it('works with single part', fromPartsTest(single, 138160))
+})