From 32bec1294ecb7cb7e88ed7dd8cc153a0f0177570 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 Jan 2020 13:06:26 +0000 Subject: [PATCH 1/2] feat: support exporting nodes encoded with the identity hash We can encode small amounts of data inside CIDs so we should be able to export them too. --- package.json | 1 - src/resolvers/identity.js | 38 ++++++++++++++++++ src/resolvers/index.js | 3 +- test/exporter-sharded.spec.js | 4 +- test/exporter.spec.js | 72 ++++++++++++++++++++++++++++++++--- 5 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 src/resolvers/identity.js diff --git a/package.json b/package.json index 27288c3..3b274bf 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "ipld-in-memory": "^3.0.0", "ipfs-unixfs-importer": "^0.42.0", "multicodec": "^1.0.0", - "multihashes": "^0.4.14", "nyc": "^15.0.0", "sinon": "^8.0.4" }, diff --git a/src/resolvers/identity.js b/src/resolvers/identity.js new file mode 100644 index 0000000..a18c6ce --- /dev/null +++ b/src/resolvers/identity.js @@ -0,0 +1,38 @@ +'use strict' + +const errCode = require('err-code') +const extractDataFromBlock = require('../utils/extract-data-from-block') +const validateOffsetAndLength = require('../utils/validate-offset-and-length') +const mh = require('multihashing-async').multihash + +const rawContent = (node) => { + return function * (options = {}) { + const { + offset, + length + } = validateOffsetAndLength(node.length, options.offset, options.length) + + yield extractDataFromBlock(node, 0, offset, offset + length) + } +} + +const resolve = async (cid, name, path, toResolve, resolve, depth, ipld) => { + if (toResolve.length) { + throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ERR_NOT_FOUND') + } + + const buf = await mh.decode(cid.multihash) + + return { + entry: { + name, + path, + cid, + node: buf, + content: rawContent(buf.digest), + depth + } + } +} + +module.exports = resolve diff --git a/src/resolvers/index.js b/src/resolvers/index.js index a3418dc..9fe34c4 100644 --- a/src/resolvers/index.js +++ b/src/resolvers/index.js @@ -5,7 +5,8 @@ const errCode = require('err-code') const resolvers = { 'dag-pb': require('./unixfs-v1'), raw: require('./raw'), - 'dag-cbor': require('./dag-cbor') + 'dag-cbor': require('./dag-cbor'), + identity: require('./identity') } const resolve = (cid, name, path, toResolve, depth, ipld) => { diff --git a/test/exporter-sharded.spec.js b/test/exporter-sharded.spec.js index 1a66215..7558580 100644 --- a/test/exporter-sharded.spec.js +++ b/test/exporter-sharded.spec.js @@ -184,7 +184,7 @@ describe('exporter sharded', function () { it('exports a file from a sharded directory inside a regular directory inside a sharded directory', async () => { const dirCid = await createShard(15) - const node = new DAGNode(new UnixFS('directory').marshal(), [ + const node = new DAGNode(new UnixFS({ type: 'directory' }).marshal(), [ new DAGLink('shard', 5, dirCid) ]) const nodeCid = await ipld.put(node, mc.DAG_PB, { @@ -192,7 +192,7 @@ describe('exporter sharded', function () { hashAlg: mh.names['sha2-256'] }) - const shardNode = new DAGNode(new UnixFS('hamt-sharded-directory').marshal(), [ + const shardNode = new DAGNode(new UnixFS({ type: 'hamt-sharded-directory' }).marshal(), [ new DAGLink('75normal-dir', 5, nodeCid) ]) const shardNodeCid = await ipld.put(shardNode, mc.DAG_PB, { diff --git a/test/exporter.spec.js b/test/exporter.spec.js index 9f50175..573c31b 100644 --- a/test/exporter.spec.js +++ b/test/exporter.spec.js @@ -12,7 +12,7 @@ const { DAGNode, DAGLink } = require('ipld-dag-pb') -const mh = require('multihashes') +const mh = require('multihashing-async').multihash const mc = require('multicodec') const exporter = require('../src') const importer = require('ipfs-unixfs-importer') @@ -38,7 +38,10 @@ describe('exporter', () => { options.content = options.content || Buffer.from([0x01, 0x02, 0x03]) options.links = options.links || [] - const file = new UnixFS(options.type, options.content) + const file = new UnixFS({ + type: options.type, + data: options.content + }) const node = new DAGNode(file.marshal(), options.links) const cid = await ipld.put(node, mc.DAG_PB, { @@ -190,7 +193,10 @@ describe('exporter', () => { it('exports a small file with links', async () => { const content = Buffer.from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - const chunk1 = new UnixFS('raw', content.slice(0, 5)) + const chunk1 = new UnixFS({ + type: 'raw', + data: content.slice(0, 5) + }) const chunkNode1 = new DAGNode(chunk1.marshal()) const chunkCid1 = await ipld.put(chunkNode1, mc.DAG_PB, { cidVersion: 0, @@ -204,7 +210,9 @@ describe('exporter', () => { hashAlg: mh.names['sha2-256'] }) - const file = new UnixFS('file') + const file = new UnixFS({ + type: 'file' + }) file.addBlockSize(5) file.addBlockSize(5) @@ -830,7 +838,9 @@ describe('exporter', () => { foo: 'bar' }, mc.DAG_CBOR) - const file = new UnixFS('file') + const file = new UnixFS({ + type: 'file' + }) file.addBlockSize(100) const cid = await ipld.put(new DAGNode(file.marshal(), [ @@ -891,4 +901,56 @@ describe('exporter', () => { expect(exported[4].name).to.equal('qux.txt') expect(exported[4].path).to.equal(`${dirCid}/qux.txt`) }) + + it('exports a CID encoded with the identity hash', async () => { + const data = Buffer.from('hello world') + const hash = mh.encode(data, 'identity') + const cid = new CID(1, 'identity', hash) + + const exported = await exporter(cid, ipld) + const result = Buffer.concat(await all(exported.content())) + + expect(result).to.deep.equal(data) + expect(result.toString('utf8')).to.equal('hello world') + }) + + it('exports a CID encoded with the identity hash with an offset', async () => { + const data = Buffer.from('hello world') + const hash = mh.encode(data, 'identity') + const cid = new CID(1, 'identity', hash) + + const exported = await exporter(cid, ipld) + const result = Buffer.concat(await all(exported.content({ + offset: 1 + }))) + + expect(result.toString('utf8')).to.equal('ello world') + }) + + it('exports a CID encoded with the identity hash with a length', async () => { + const data = Buffer.from('hello world') + const hash = mh.encode(data, 'identity') + const cid = new CID(1, 'identity', hash) + + const exported = await exporter(cid, ipld) + const result = Buffer.concat(await all(exported.content({ + length: 1 + }))) + + expect(result.toString('utf8')).to.equal('h') + }) + + it('exports a CID encoded with the identity hash with an offset and a length', async () => { + const data = Buffer.from('hello world') + const hash = mh.encode(data, 'identity') + const cid = new CID(1, 'identity', hash) + + const exported = await exporter(cid, ipld) + const result = Buffer.concat(await all(exported.content({ + offset: 3, + length: 1 + }))) + + expect(result.toString('utf8')).to.equal('l') + }) }) From 7c902f3f0bf9e437b344ef398afc344e99b17f01 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 Jan 2020 13:33:24 +0000 Subject: [PATCH 2/2] chore: update require --- test/exporter-sharded.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/exporter-sharded.spec.js b/test/exporter-sharded.spec.js index 7558580..6d247ef 100644 --- a/test/exporter-sharded.spec.js +++ b/test/exporter-sharded.spec.js @@ -7,7 +7,7 @@ const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') const UnixFS = require('ipfs-unixfs') -const mh = require('multihashes') +const mh = require('multihashing-async').multihash const mc = require('multicodec') const all = require('async-iterator-all') const last = require('async-iterator-last')