From e83f95a8ac86b2e350fe9addd837e2c6551dc282 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 24 Jun 2019 15:39:08 -0400 Subject: [PATCH] fix: use offline blockstore for pin operations --- src/core/components/dag.js | 30 ++++++++++++++++++++---------- src/core/components/pin.js | 15 ++++++++++++++- src/core/index.js | 2 +- test/core/pin.js | 16 ++++++++++++++-- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/core/components/dag.js b/src/core/components/dag.js index 5e750c35b0..81a4d7b196 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -10,8 +10,8 @@ const flattenDeep = require('just-flatten-it') const errCode = require('err-code') const multicodec = require('multicodec') -module.exports = function dag (self) { - return { +module.exports = function dag (self, ipld) { + const dag = { put: promisify((dagNode, options, callback) => { if (typeof options === 'function') { callback = options @@ -53,7 +53,7 @@ module.exports = function dag (self) { } } - self._ipld.put(dagNode, options.format, { + ipld.put(dagNode, options.format, { hashAlg: options.hashAlg, cidVersion: options.version }).then( @@ -115,22 +115,30 @@ module.exports = function dag (self) { self._preload(cid) } + const niceError = (err) => { + if (err.code === 'ERR_NOT_FOUND') { + err.name = 'ERR_NOT_FOUND' + err.message = `Could not resolve node with CID ${cid} - ${err.message}` + } + return err + } + if (path === undefined || path === '/') { - self._ipld.get(cid).then( + ipld.get(cid).then( (value) => { callback(null, { value, remainderPath: '' }) }, - (error) => callback(error) + (error) => callback(niceError(error)) ) } else { - const result = self._ipld.resolve(cid, path) + const result = ipld.resolve(cid, path) const promisedValue = options.localResolve ? result.first() : result.last() promisedValue.then( (value) => callback(null, value), - (error) => callback(error) + (error) => callback(niceError(error)) ) } }), @@ -177,7 +185,7 @@ module.exports = function dag (self) { } pull( - iterToPull(self._ipld.tree(cid, path, options)), + iterToPull(ipld.tree(cid, path, options)), pull.collect(callback) ) }), @@ -201,11 +209,11 @@ module.exports = function dag (self) { return setImmediate(() => callback(errCode(err, 'ERR_INVALID_CID'))) } - self.dag.get(cid, '', options, (err, res) => { + dag.get(cid, '', options, (err, res) => { if (err) { return callback(err) } mapAsync(res.value.Links, (link, cb) => { - self.dag._getRecursive(link.Hash, options, cb) + dag._getRecursive(link.Hash, options, cb) }, (err, nodes) => { // console.log('nodes:', nodes) if (err) return callback(err) @@ -214,4 +222,6 @@ module.exports = function dag (self) { }) }) } + + return dag } diff --git a/src/core/components/pin.js b/src/core/components/pin.js index 40d661e9a0..70e4085a21 100644 --- a/src/core/components/pin.js +++ b/src/core/components/pin.js @@ -9,8 +9,12 @@ const waterfall = require('async/waterfall') const setImmediate = require('async/setImmediate') const errCode = require('err-code') const multibase = require('multibase') +const Ipld = require('ipld') +const BlockService = require('ipfs-block-service') const { resolvePath } = require('../utils') +const ipldOptions = require('../runtime/ipld-nodejs') +const components = require('../components') const PinManager = require('./pin/pin-manager') const PinTypes = PinManager.PinTypes @@ -18,8 +22,17 @@ function toB58String (hash) { return new CID(hash).toBaseEncodedString() } +// The pinner uses BlockService to store pin sets internally. We don't want +// to expose these internal blocks to the network, so we create an offline +// BlockService (one that doesn't talk to Bitswap) +function createOfflineDag (self) { + const offlineBlockService = new BlockService(self._repo) + const offlineIpld = new Ipld(ipldOptions(offlineBlockService, self._options.ipld, self.log)) + return components.dag(self, offlineIpld) +} + module.exports = (self) => { - const dag = self.dag + const dag = createOfflineDag(self) const pinManager = new PinManager(self._repo, dag, self._options.repoOwner, self.log) const pin = { diff --git a/src/core/index.js b/src/core/index.js index f8f2b53b16..3c2af98c98 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -99,7 +99,7 @@ class IPFS extends EventEmitter { this.config = components.config(this) this.block = components.block(this) this.object = components.object(this) - this.dag = components.dag(this) + this.dag = components.dag(this, this._ipld) this.files = components.filesMFS(this) this.libp2p = null // assigned on start this.swarm = components.swarm(this) diff --git a/test/core/pin.js b/test/core/pin.js index b23b5aeb8a..512f8adff6 100644 --- a/test/core/pin.js +++ b/test/core/pin.js @@ -8,6 +8,7 @@ const expect = chai.expect chai.use(dirtyChai) const fs = require('fs') +const sinon = require('sinon') const IPFS = require('../../src/core') const createTempRepo = require('../utils/create-repo-nodejs') @@ -190,9 +191,10 @@ describe('pin', function () { }) it('can\'t pin item not in datastore', function () { - this.timeout(5 * 1000) const falseHash = `${pins.root.slice(0, -2)}ss` - return expectTimeout(pin.add(falseHash), 4000) + return pin.add(falseHash) + .then(() => expect.fail('Expected ERR_NOT_FOUND error')) + .catch(err => expect(err.code).to.equal('ERR_NOT_FOUND')) }) // TODO block rm breaks subsequent tests @@ -317,6 +319,11 @@ describe('pin', function () { beforeEach(function () { return pin.add(pins.root) }) + before(() => { + sinon.spy(ipfs._bitswap, 'put') + sinon.spy(ipfs._bitswap, 'putMany') + }) + after(() => sinon.restore()) it('flushes', function () { return pin.ls() @@ -330,6 +337,11 @@ describe('pin', function () { .then(() => pin.ls()) .then(ls => expect(ls.length).to.eql(1)) }) + + it('does not put to bitswap', function () { + expect(ipfs._bitswap.put.called).to.equal(false) + expect(ipfs._bitswap.putMany.called).to.equal(false) + }) }) describe('locking', function () {