From c5405d18c07f64d756445c8df9d5940788d0c833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Thu, 3 Oct 2019 14:34:49 +0200 Subject: [PATCH] feat: store blocks under multihash key BREAKING CHANGE: repo.blocks.query() now returns multihashes as a key instead of CID. If you want to have CID returned call it as query({}, true), which will constructs CIDv1 using IPLD's RAW codec. This means that this constructed CID might not equal to the one that the block was originally saved. Related to https://github.com/ipfs/js-ipfs/issues/2415 --- README.md | 19 ++++++++ src/blockstore-utils.js | 45 ++++++++++++++++--- src/blockstore.js | 84 ++++++++++++++--------------------- src/constants.js | 2 +- src/index.js | 6 +-- test/blockstore-test.js | 7 ++- test/blockstore-utils-test.js | 5 ++- 7 files changed, 105 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index f5e46902..07f05941 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,25 @@ Get block. * `cid` is the content id of [type CID](https://github.com/ipld/js-cid#readme). +#### `Promise repo.blocks.has (obj)` + +Indicate if block is present + +* `obj` is either the content id of [type CID](https://github.com/ipld/js-cid#readme) or [multihash](https://github.com/multiformats/js-multihashing). + +#### `Promise repo.blocks.delete (obj)` + +Deletes + +* `obj` is either the content id of [type CID](https://github.com/ipld/js-cid#readme) or [multihash](https://github.com/multiformats/js-multihashing). + +#### `Promise> repo.blocks.query (query, reconstructsCids)` + +Query what blocks are available in blockstore. + +* `query` is a object as specified in [interface-datastore](https://github.com/ipfs/interface-datastore#query). +* `reconstructsCids` a flag defining if the block's key is a reconstructed CID (eq. CIDv1 with RAW IPLD codec) or multihash + Datastore: #### `repo.datastore` diff --git a/src/blockstore-utils.js b/src/blockstore-utils.js index b1d00f74..06be7fee 100644 --- a/src/blockstore-utils.js +++ b/src/blockstore-utils.js @@ -7,23 +7,58 @@ const CID = require('cids') /** * Transform a cid to the appropriate datastore key. * - * @param {CID} cid + * @param {Buffer} multihash * @returns {Key} */ -exports.cidToKey = cid => { +exports.multihashToKey = multihash => { const enc = new base32.Encoder() - return new Key('/' + enc.write(cid.buffer).finalize(), false) + return new Key('/' + enc.write(multihash).finalize(), false) } /** * Transform a datastore Key instance to a CID + * As Key is a multihash of the CID, it is reconstructed using IPLD's RAW codec. + * Hence it is highly probable that stored CID will differ from a CID retrieved from blockstore. * * @param {Key} key * @returns {CID} */ -exports.keyToCid = key => { + +function keyToCid (key) { + // Block key is of the form / + const decoder = new base32.Decoder() + const buff = decoder.write(key.toString().slice(1)).finalize() + return new CID(1, 'raw', Buffer.from(buff)) +} + +exports.keyToCid = keyToCid + +/** + * Transform a datastore Key instance to a multihash instance. + * + * @param {Key} key + * @returns {Buffer} + */ + +function keyToMultihash (key) { // Block key is of the form / const decoder = new base32.Decoder() const buff = decoder.write(key.toString().slice(1)).finalize() - return new CID(Buffer.from(buff)) + return Buffer.from(buff) +} + +exports.keyToMultihash = keyToMultihash + +/** + * Transforms a datastore Key containing multihash to a Key that contains reconstructed CID + * + * @param {Key} key + * @returns {CID} + */ +function keyToCidKey (key) { + const cid = keyToCid(key) + const enc = new base32.Encoder() + return new Key('/' + enc.write(cid.buffer).finalize(), false) } + +exports.keyToCidKey = keyToCidKey diff --git a/src/blockstore.js b/src/blockstore.js index 27fc5415..f1d4cfa5 100644 --- a/src/blockstore.js +++ b/src/blockstore.js @@ -5,7 +5,7 @@ const ShardingStore = core.ShardingDatastore const Block = require('ipfs-block') const CID = require('cids') const errcode = require('err-code') -const { cidToKey } = require('./blockstore-utils') +const { multihashToKey, keyToCidKey } = require('./blockstore-utils') module.exports = async (filestore, options) => { const store = await maybeWithSharding(filestore, options) @@ -26,10 +26,15 @@ function createBaseStore (store) { * Query the store. * * @param {object} query + * @param {boolean} reconstructsCids - Defines if Keys are converted to a reconstructed CID using IPLD_RAW codec * @return {Iterable} */ - async * query (query) { + async * query (query, reconstructsCids = false) { for await (const block of store.query(query)) { + if (reconstructsCids) { + block.key = keyToCidKey(block.key) + } + yield block } }, @@ -43,27 +48,9 @@ function createBaseStore (store) { if (!CID.isCID(cid)) { throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID') } - const key = cidToKey(cid) - let blockData - try { - blockData = await store.get(key) - return new Block(blockData, cid) - } catch (err) { - if (err.code === 'ERR_NOT_FOUND') { - const otherCid = cidToOtherVersion(cid) - - if (!otherCid) { - throw err - } - - const otherKey = cidToKey(otherCid) - const blockData = await store.get(otherKey) - await store.put(key, blockData) - return new Block(blockData, cid) - } - - throw err - } + const key = multihashToKey(cid.multihash) + const blockData = await store.get(key) + return new Block(blockData, cid) }, /** * Write a single block to the store. @@ -76,7 +63,7 @@ function createBaseStore (store) { throw new Error('invalid block') } - const k = cidToKey(block.cid) + const k = multihashToKey(block.cid.multihash) const exists = await store.has(k) if (exists) return return store.put(k, block.data) @@ -90,7 +77,7 @@ function createBaseStore (store) { */ async putMany (blocks) { const keys = blocks.map((b) => ({ - key: cidToKey(b.cid), + key: multihashToKey(b.cid.multihash), block: b })) @@ -109,33 +96,38 @@ function createBaseStore (store) { return batch.commit() }, /** - * Does the store contain block with this cid? + * Does the store contain block with this multihash or CID? * - * @param {CID} cid - * @returns {Promise} + * @param {CID|Buffer} obj + * @returns {Promise} */ - async has (cid) { - if (!CID.isCID(cid)) { - throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID') + has (obj) { + if (CID.isCID(obj)) { + obj = obj.multihash } - const exists = await store.has(cidToKey(cid)) - if (exists) return exists - const otherCid = cidToOtherVersion(cid) - if (!otherCid) return false - return store.has(cidToKey(otherCid)) + if(!Buffer.isBuffer(obj)){ + throw errcode(new Error('Not a valid key'), 'ERR_INVALID_KEY') + } + + return store.has(multihashToKey(obj)) }, /** - * Delete a block from the store + * Delete a CID or multihash from the store * - * @param {CID} cid + * @param {CID|Buffer} obj * @returns {Promise} */ - async delete (cid) { // eslint-disable-line require-await - if (!CID.isCID(cid)) { - throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID') + async delete (obj) { // eslint-disable-line require-await + if (CID.isCID(obj)) { + obj = obj.multihash } - return store.delete(cidToKey(cid)) + + if(!Buffer.isBuffer(obj)){ + throw errcode(new Error('Not a valid key'), 'ERR_INVALID_KEY') + } + + return store.delete(multihashToKey(obj)) }, /** * Close the store @@ -147,11 +139,3 @@ function createBaseStore (store) { } } } - -function cidToOtherVersion (cid) { - try { - return cid.version === 0 ? cid.toV1() : cid.toV0() - } catch (err) { - return null - } -} diff --git a/src/constants.js b/src/constants.js index d680f22a..67c1643d 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,5 +1,5 @@ 'use strict' module.exports = { - repoVersion: 7 + repoVersion: 8 } diff --git a/src/index.js b/src/index.js index 5b203be2..6efbd5cc 100644 --- a/src/index.js +++ b/src/index.js @@ -277,11 +277,11 @@ class IpfsRepo { let count = new Big(0) let size = new Big(0) - for await (const block of this.blocks.query({})) { + for await (const block of this.blocks.query({}, false)) { count = count.plus(1) size = size .plus(block.value.byteLength) - .plus(block.key._buf.byteLength) + .plus(block.key.toBuffer().byteLength) } return { count, size } @@ -292,7 +292,7 @@ async function getSize (queryFn) { let sum = new Big(0) for await (const block of queryFn.query({})) { sum.plus(block.value.byteLength) - .plus(block.key._buf.byteLength) + .plus(block.key.toBuffer().byteLength) } return sum } diff --git a/test/blockstore-test.js b/test/blockstore-test.js index 07ca44e6..255f9387 100644 --- a/test/blockstore-test.js +++ b/test/blockstore-test.js @@ -85,9 +85,11 @@ module.exports = (repo) => { close () { } + has () { return true } + batch () { return { put () { @@ -217,6 +219,7 @@ module.exports = (repo) => { close () { } + get (c) { if (c.toString() === key.toString()) { throw err @@ -278,7 +281,7 @@ module.exports = (repo) => { await repo.blocks.has('foo') throw new Error('Should have thrown') } catch (err) { - expect(err.code).to.equal('ERR_INVALID_CID') + expect(err.code).to.equal('ERR_INVALID_KEY') } }) @@ -304,7 +307,7 @@ module.exports = (repo) => { await repo.blocks.delete('foo') throw new Error('Should have thrown') } catch (err) { - expect(err.code).to.equal('ERR_INVALID_CID') + expect(err.code).to.equal('ERR_INVALID_KEY') } }) }) diff --git a/test/blockstore-utils-test.js b/test/blockstore-utils-test.js index 3bea8544..9d9f3698 100644 --- a/test/blockstore-utils-test.js +++ b/test/blockstore-utils-test.js @@ -11,8 +11,9 @@ const Repo = require('../src') module.exports = () => { describe('blockstore utils', () => { it('converts a CID to a datastore Key and back', () => { - const originalCid = new CID('Qme6KJdKcp85TYbLxuLV7oQzMiLremD7HMoXLZEmgo6Rnh') - const key = Repo.utils.blockstore.cidToKey(originalCid) + // CIDv1 in base32 with IPLD raw codec + const originalCid = new CID('bafkreihkb3vrxxex5zvzkr3s3a6noe223r7jka4ofjy2nkzu27kueg76ii') + const key = Repo.utils.blockstore.multihashToKey(originalCid.multihash) expect(key instanceof Key).to.be.true() const cid = Repo.utils.blockstore.keyToCid(key) expect(cid instanceof CID).to.be.true()