From eb1f356feaafab3da1ba61d266d99504ec63a417 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 4 Sep 2017 10:31:31 +0100 Subject: [PATCH 1/4] WIP support support specify hash alg --- src/files/add.js | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/files/add.js b/src/files/add.js index d5aac24c5..459302675 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -21,7 +21,27 @@ module.exports = (send) => { return callback(new Error('"files" must be a buffer, readable stream, or array of objects')) } - const request = { path: 'add', files: files, qs: opts } + const qs = {} + + if (opts['cid-version'] != null) { + qs['cid-version'] = opts['cid-version'] + } else if (opts.cidVersion != null) { + qs['cid-version'] = opts.cidVersion + } + + if (opts['raw-leaves'] != null) { + qs['raw-leaves'] = opts['raw-version'] + } else if (opts.rawLeaves != null) { + qs['raw-leaves'] = opts.rawLeaves + } + + if (opts.hash != null) { + qs.hash = opts.hash + } else if (opts.hashAlg != null) { + qs.hash = opts.hashAlg + } + + const request = { path: 'add', files: files, qs: qs } // Transform the response stream to DAGNode values const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback) From 1b5accc58cd56d25bbe71f5bf6486f40375d3c90 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 6 Sep 2017 10:55:50 +0100 Subject: [PATCH 2/4] Add test for add with --hash option. Pass raw cid/multihash to ipfs object get/data --- src/files/add.js | 2 +- src/utils/get-dagnode.js | 10 ++------ test/files.spec.js | 51 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/files/add.js b/src/files/add.js index 459302675..81e8734f4 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -30,7 +30,7 @@ module.exports = (send) => { } if (opts['raw-leaves'] != null) { - qs['raw-leaves'] = opts['raw-version'] + qs['raw-leaves'] = opts['raw-leaves'] } else if (opts.rawLeaves != null) { qs['raw-leaves'] = opts.rawLeaves } diff --git a/src/utils/get-dagnode.js b/src/utils/get-dagnode.js index edb89c4d9..4bcd5d8fd 100644 --- a/src/utils/get-dagnode.js +++ b/src/utils/get-dagnode.js @@ -2,22 +2,16 @@ const DAGNode = require('ipld-dag-pb').DAGNode const parallel = require('async/parallel') -const CID = require('cids') -const mh = require('multihashes') const streamToValue = require('./stream-to-value') module.exports = function (send, hash, callback) { - // Until js-ipfs supports object/get and object/data by CID - // we need to convert our CID or multihash hash into a multihash - const multihash = mh.toB58String(new CID(hash).multihash) - // Retrieve the object and its data in parallel, then produce a DAGNode // instance using this information. parallel([ (done) => { send({ path: 'object/get', - args: multihash + args: hash }, done) }, (done) => { @@ -26,7 +20,7 @@ module.exports = function (send, hash, callback) { // See https://github.com/ipfs/go-ipfs/issues/1582 for more details. send({ path: 'object/data', - args: multihash + args: hash }, done) } ], (err, res) => { diff --git a/test/files.spec.js b/test/files.spec.js index 8ea2f9f58..629e5e9b0 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -8,6 +8,8 @@ const expect = chai.expect chai.use(dirtyChai) const isNode = require('detect-node') const loadFixture = require('aegir/fixtures') +const mh = require('multihashes') +const CID = require('cids') const FactoryClient = require('./ipfs-factory/client') @@ -15,6 +17,18 @@ const testfile = isNode ? loadFixture(__dirname, '/fixtures/testfile.txt') : loadFixture(__dirname, 'fixtures/testfile.txt') +// TODO: Test against all algorithms Object.keys(mh.names) +// This subset is known to work with both go-ipfs and js-ipfs as of 2017-09-05 +const HASH_ALGS = [ + 'sha1', + 'sha2-256', + 'sha2-512', + 'keccak-224', + 'keccak-256', + 'keccak-384', + 'keccak-512' +] + describe('.files (the MFS API part)', function () { this.timeout(120 * 1000) @@ -73,6 +87,25 @@ describe('.files (the MFS API part)', function () { }) }) + HASH_ALGS.forEach((name) => { + it(`files.add with hash=${name} and raw-leaves=false`, (done) => { + const content = String(Math.random() + Date.now()) + const file = { + path: content + '.txt', + content: Buffer.from(content) + } + const options = { hash: name, 'raw-leaves': false } + + ipfs.files.add([file], options, (err, res) => { + if (err) return done(err) + expect(res).to.have.length(1) + const cid = new CID(res[0].hash) + expect(mh.decode(cid.multihash).name).to.equal(name) + done() + }) + }) + }) + it('files.mkdir', (done) => { ipfs.files.mkdir('/test-folder', done) }) @@ -230,6 +263,24 @@ describe('.files (the MFS API part)', function () { }) }) + HASH_ALGS.forEach((name) => { + it(`files.add with hash=${name} and raw-leaves=false`, () => { + const content = String(Math.random() + Date.now()) + const file = { + path: content + '.txt', + content: Buffer.from(content) + } + const options = { hash: name, 'raw-leaves': false } + + return ipfs.files.add([file], options) + .then((res) => { + expect(res).to.have.length(1) + const cid = new CID(res[0].hash) + expect(mh.decode(cid.multihash).name).to.equal(name) + }) + }) + }) + it('files.mkdir', () => { return ipfs.files.mkdir('/test-folder') }) From 19fcd7b3871596e5c6f87ed5b9a22e3d9ce14a6e Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 7 Sep 2017 17:32:09 +0100 Subject: [PATCH 3/4] Allow object get/data to accept CID --- src/object/data.js | 13 ++++++++----- src/object/get.js | 15 +++++++++------ src/utils/get-dagnode.js | 13 +++++++++++-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/object/data.js b/src/object/data.js index 9ba2ef100..1f0ebb55a 100644 --- a/src/object/data.js +++ b/src/object/data.js @@ -2,7 +2,7 @@ const promisify = require('promisify-es6') const streamToValue = require('../utils/stream-to-value') -const cleanMultihash = require('../utils/clean-multihash') +const CID = require('cids') const LRU = require('lru-cache') const lruOptions = { max: 128 @@ -11,7 +11,7 @@ const lruOptions = { const cache = LRU(lruOptions) module.exports = (send) => { - return promisify((multihash, options, callback) => { + return promisify((hash, options, callback) => { if (typeof options === 'function') { callback = options options = {} @@ -20,13 +20,16 @@ module.exports = (send) => { options = {} } + let cid, b58Hash + try { - multihash = cleanMultihash(multihash, options) + cid = new CID(hash) + b58Hash = cid.toBaseEncodedString() } catch (err) { return callback(err) } - const node = cache.get(multihash) + const node = cache.get(b58Hash) if (node) { return callback(null, node.data) @@ -34,7 +37,7 @@ module.exports = (send) => { send({ path: 'object/data', - args: multihash + args: b58Hash }, (err, result) => { if (err) { return callback(err) diff --git a/src/object/get.js b/src/object/get.js index 5f0f80e10..4649a9c74 100644 --- a/src/object/get.js +++ b/src/object/get.js @@ -5,7 +5,7 @@ const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const DAGLink = dagPB.DAGLink const bs58 = require('bs58') -const cleanMultihash = require('../utils/clean-multihash') +const CID = require('cids') const LRU = require('lru-cache') const lruOptions = { max: 128 @@ -14,7 +14,7 @@ const lruOptions = { const cache = LRU(lruOptions) module.exports = (send) => { - return promisify((multihash, options, callback) => { + return promisify((hash, options, callback) => { if (typeof options === 'function') { callback = options options = {} @@ -24,13 +24,16 @@ module.exports = (send) => { options = {} } + let cid, b58Hash + try { - multihash = cleanMultihash(multihash, options) + cid = new CID(hash) + b58Hash = cid.toBaseEncodedString() } catch (err) { return callback(err) } - const node = cache.get(multihash) + const node = cache.get(b58Hash) if (node) { return callback(null, node) @@ -38,7 +41,7 @@ module.exports = (send) => { send({ path: 'object/get', - args: multihash + args: b58Hash }, (err, result) => { if (err) { return callback(err) @@ -52,7 +55,7 @@ module.exports = (send) => { if (err) { return callback(err) } - cache.set(multihash, node) + cache.set(b58Hash, node) callback(null, node) }) }) diff --git a/src/utils/get-dagnode.js b/src/utils/get-dagnode.js index 4bcd5d8fd..1cfd1dcd7 100644 --- a/src/utils/get-dagnode.js +++ b/src/utils/get-dagnode.js @@ -2,16 +2,25 @@ const DAGNode = require('ipld-dag-pb').DAGNode const parallel = require('async/parallel') +const CID = require('cids') const streamToValue = require('./stream-to-value') module.exports = function (send, hash, callback) { + let cid + + try { + cid = new CID(hash) + } catch (err) { + return callback(err) + } + // Retrieve the object and its data in parallel, then produce a DAGNode // instance using this information. parallel([ (done) => { send({ path: 'object/get', - args: hash + args: cid.toBaseEncodedString() }, done) }, (done) => { @@ -20,7 +29,7 @@ module.exports = function (send, hash, callback) { // See https://github.com/ipfs/go-ipfs/issues/1582 for more details. send({ path: 'object/data', - args: hash + args: cid.toBaseEncodedString() }, done) } ], (err, res) => { From ea96f21f5e465e892a3d39fe95c70dfb068c8a53 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 8 Sep 2017 12:31:17 +0100 Subject: [PATCH 4/4] Var naming tweaks from review --- src/object/data.js | 12 ++++++------ src/object/get.js | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/object/data.js b/src/object/data.js index 1f0ebb55a..ace3cc992 100644 --- a/src/object/data.js +++ b/src/object/data.js @@ -11,7 +11,7 @@ const lruOptions = { const cache = LRU(lruOptions) module.exports = (send) => { - return promisify((hash, options, callback) => { + return promisify((cid, options, callback) => { if (typeof options === 'function') { callback = options options = {} @@ -20,16 +20,16 @@ module.exports = (send) => { options = {} } - let cid, b58Hash + let cidB58Str try { - cid = new CID(hash) - b58Hash = cid.toBaseEncodedString() + cid = new CID(cid) + cidB58Str = cid.toBaseEncodedString() } catch (err) { return callback(err) } - const node = cache.get(b58Hash) + const node = cache.get(cidB58Str) if (node) { return callback(null, node.data) @@ -37,7 +37,7 @@ module.exports = (send) => { send({ path: 'object/data', - args: b58Hash + args: cidB58Str }, (err, result) => { if (err) { return callback(err) diff --git a/src/object/get.js b/src/object/get.js index 4649a9c74..82b21bb58 100644 --- a/src/object/get.js +++ b/src/object/get.js @@ -14,7 +14,7 @@ const lruOptions = { const cache = LRU(lruOptions) module.exports = (send) => { - return promisify((hash, options, callback) => { + return promisify((cid, options, callback) => { if (typeof options === 'function') { callback = options options = {} @@ -24,16 +24,16 @@ module.exports = (send) => { options = {} } - let cid, b58Hash + let cidB58Str try { - cid = new CID(hash) - b58Hash = cid.toBaseEncodedString() + cid = new CID(cid) + cidB58Str = cid.toBaseEncodedString() } catch (err) { return callback(err) } - const node = cache.get(b58Hash) + const node = cache.get(cidB58Str) if (node) { return callback(null, node) @@ -41,7 +41,7 @@ module.exports = (send) => { send({ path: 'object/get', - args: b58Hash + args: cidB58Str }, (err, result) => { if (err) { return callback(err) @@ -55,7 +55,7 @@ module.exports = (send) => { if (err) { return callback(err) } - cache.set(b58Hash, node) + cache.set(cidB58Str, node) callback(null, node) }) })