From 30f8b06298df9b2d4b8d6aea74720ecfb17f504e Mon Sep 17 00:00:00 2001 From: Mikeal Rogers Date: Sat, 11 Jan 2020 00:07:21 +0000 Subject: [PATCH 1/4] feat: support custom chunker --- src/chunker/index.js | 7 ++++++- test/chunker-custom.spec.js | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 test/chunker-custom.spec.js diff --git a/src/chunker/index.js b/src/chunker/index.js index ec2c494..71ca417 100644 --- a/src/chunker/index.js +++ b/src/chunker/index.js @@ -8,7 +8,12 @@ const chunkers = { } module.exports = (type, source, options) => { - const chunker = chunkers[type] + let chunker + if (typeof type === 'function') { + chunker = type + } else { + chunker = chunkers[type] + } if (!chunker) { throw errCode(new Error(`Unknkown chunker named ${type}`), 'ERR_UNKNOWN_CHUNKER') diff --git a/test/chunker-custom.spec.js b/test/chunker-custom.spec.js new file mode 100644 index 0000000..76263ed --- /dev/null +++ b/test/chunker-custom.spec.js @@ -0,0 +1,41 @@ +/* eslint-env mocha */ +'use strict' + +const importer = require('../src') + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const spy = require('sinon/lib/sinon/spy') +const IPLD = require('ipld') +const inMemory = require('ipld-in-memory') +const UnixFs = require('ipfs-unixfs') +const collectLeafCids = require('./helpers/collect-leaf-cids') +const loadFixture = require('aegir/fixtures') +const isNode = require('detect-node') +const bigFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1.2MiB.txt') +const smallFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/200Bytes.txt') +const all = require('it-all') +const first = require('it-first') + +const iter = async function * () { + yield Buffer.from('one') + yield Buffer.from('two') +} + +describe('custom chunker', function () { + it('keeps custom chunking', async () => { + const chunker = source => source + const content = iter() + const inmem = await inMemory(IPLD) + const sizes = [11, 11, 116] + const ipld = { put: (node, ...args) => { + expect(node.toJSON().size).to.equal(sizes.shift()) + return inmem.put(node, ...args) + }} + for await (const part of importer([{path: 'test', content}], ipld, {chunker})) { + expect(part.size).to.equal(116) + } + expect(sizes).to.be.empty + }) +}) From 327b5db7d9fd77332ea7b326ac2c1546f0761f06 Mon Sep 17 00:00:00 2001 From: Mikeal Rogers Date: Sat, 11 Jan 2020 00:10:16 +0000 Subject: [PATCH 2/4] fix: make the linter happy --- test/chunker-custom.spec.js | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/test/chunker-custom.spec.js b/test/chunker-custom.spec.js index 76263ed..9940fe2 100644 --- a/test/chunker-custom.spec.js +++ b/test/chunker-custom.spec.js @@ -6,17 +6,8 @@ const importer = require('../src') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const spy = require('sinon/lib/sinon/spy') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const UnixFs = require('ipfs-unixfs') -const collectLeafCids = require('./helpers/collect-leaf-cids') -const loadFixture = require('aegir/fixtures') -const isNode = require('detect-node') -const bigFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1.2MiB.txt') -const smallFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/200Bytes.txt') -const all = require('it-all') -const first = require('it-first') const iter = async function * () { yield Buffer.from('one') @@ -29,13 +20,15 @@ describe('custom chunker', function () { const content = iter() const inmem = await inMemory(IPLD) const sizes = [11, 11, 116] - const ipld = { put: (node, ...args) => { - expect(node.toJSON().size).to.equal(sizes.shift()) - return inmem.put(node, ...args) - }} - for await (const part of importer([{path: 'test', content}], ipld, {chunker})) { + const ipld = { + put: (node, ...args) => { + expect(node.toJSON().size).to.equal(sizes.shift()) + return inmem.put(node, ...args) + } + } + for await (const part of importer([{ path: 'test', content }], ipld, { chunker })) { expect(part.size).to.equal(116) } - expect(sizes).to.be.empty + expect(sizes).to.be.empty() }) }) From 8b858a464cd185c214450329844005a75209f82e Mon Sep 17 00:00:00 2001 From: Mikeal Rogers Date: Sat, 11 Jan 2020 00:16:58 +0000 Subject: [PATCH 3/4] test: disable linter for async generator line --- test/chunker-custom.spec.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/chunker-custom.spec.js b/test/chunker-custom.spec.js index 9940fe2..350ce93 100644 --- a/test/chunker-custom.spec.js +++ b/test/chunker-custom.spec.js @@ -9,6 +9,8 @@ const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') +// eslint bug https://github.com/eslint/eslint/issues/12459 +// eslint-disable-next-line require-await const iter = async function * () { yield Buffer.from('one') yield Buffer.from('two') From 040c0d9f8078b5a2f73a14e0ba61bcc7c5fe35e4 Mon Sep 17 00:00:00 2001 From: Mikeal Rogers Date: Sat, 11 Jan 2020 01:35:51 +0000 Subject: [PATCH 4/4] feat: build the DAG for a file from generator of CIDs and sizes --- src/dag-builder/file/index.js | 35 ++++++++++++++++------------ src/dag-builder/index.js | 9 +++++++- test/fromParts.spec.js | 43 +++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 16 deletions(-) create mode 100644 test/fromParts.spec.js diff --git a/src/dag-builder/file/index.js b/src/dag-builder/file/index.js index f9df4fd..d3f3514 100644 --- a/src/dag-builder/file/index.js +++ b/src/dag-builder/file/index.js @@ -27,23 +27,28 @@ async function * importBuffer (file, source, ipld, options) { ...options } - if (options.rawLeaves) { - node = buffer - - opts.codec = 'raw' - opts.cidVersion = 1 + let cid + if (options.fromParts) { + node = { length: buffer.size } + cid = buffer.cid } else { - unixfs = new UnixFS({ - type: options.leafType, - data: buffer, - mtime: file.mtime, - mode: file.mode - }) - - node = new DAGNode(unixfs.marshal()) - } + if (options.rawLeaves) { + node = buffer - const cid = await persist(node, ipld, opts) + opts.codec = 'raw' + opts.cidVersion = 1 + } else { + unixfs = new UnixFS({ + type: options.leafType, + data: buffer, + mtime: file.mtime, + mode: file.mode + }) + + node = new DAGNode(unixfs.marshal()) + } + cid = await persist(node, ipld, opts) + } return { cid: cid, diff --git a/src/dag-builder/index.js b/src/dag-builder/index.js index bc2f4b2..4e60a93 100644 --- a/src/dag-builder/index.js +++ b/src/dag-builder/index.js @@ -30,7 +30,14 @@ async function * dagBuilder (source, ipld, options) { } } - const chunker = createChunker(options.chunker, validateChunks(source), options) + if (options.fromParts) { + options.rawLeaves = true + options.chunker = source => source + options.reduceSingleLeafToSelf = false + } else { + source = validateChunks(source) + } + const chunker = createChunker(options.chunker, source, options) // item is a file yield () => fileBuilder(entry, chunker, ipld, options) diff --git a/test/fromParts.spec.js b/test/fromParts.spec.js new file mode 100644 index 0000000..98073bb --- /dev/null +++ b/test/fromParts.spec.js @@ -0,0 +1,43 @@ +/* eslint-env mocha */ +'use strict' + +const importer = require('../src') + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const IPLD = require('ipld') +const inMemory = require('ipld-in-memory') +const CID = require('cids') + +const fromPartsTest = (iter, size) => async () => { + const content = iter() + const inmem = await inMemory(IPLD) + const sizes = [size] + const ipld = { + put: (node, ...args) => { + expect(node.toJSON().size).to.equal(sizes.shift()) + return inmem.put(node, ...args) + } + } + for await (const part of importer([{ path: 'test', content }], ipld, { fromParts: true })) { + expect(part.size).to.equal(size) + } + expect(sizes).to.be.empty() +} + +describe('custom chunker', function () { + // eslint bug https://github.com/eslint/eslint/issues/12459 + // eslint-disable-next-line require-await + const multi = async function * () { + yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') } + yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') } + } + it('works with multiple parts', fromPartsTest(multi, 276312)) + + // eslint-disable-next-line require-await + const single = async function * () { + yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') } + } + it('works with single part', fromPartsTest(single, 138160)) +})