feat: store blocks under multihash key

BREAKING CHANGE: repo.blocks.query() now returns multihashes as a key instead of CID. If you want to have CID returned call it as query({}, true), which will constructs CIDv1 using IPLD's RAW codec. This means that this constructed CID might not equal to the one that the block was originally saved. Related to ipfs/js-ipfs#2415
ipfs · Oct 9, 2019 · 969f41a · 969f41a
1 parent 62a0aa8
commit 969f41a
Show file tree

Hide file tree

Showing 7 changed files with 105 additions and 63 deletions.
diff --git a/README.md b/README.md
@@ -197,6 +197,25 @@ Get block.
 
 * `cid` is the content id of [type CID](https://github.com/ipld/js-cid#readme).
 
+#### `Promise<boolean> repo.blocks.has (obj)`
+
+Indicate if block is present
+
+* `obj` is either the content id of [type CID](https://github.com/ipld/js-cid#readme) or [multihash](https://github.com/multiformats/js-multihashing).
+
+#### `Promise<boolean> repo.blocks.delete (obj)`
+
+Deletes
+
+* `obj` is either the content id of [type CID](https://github.com/ipld/js-cid#readme) or [multihash](https://github.com/multiformats/js-multihashing).
+
+#### `Promise<Array<Object>> repo.blocks.query (query, reconstructsCids)`
+
+Query what blocks are available in blockstore.
+
+* `query` is a object as specified in [interface-datastore](https://github.com/ipfs/interface-datastore#query).
+* `reconstructsCids` a flag defining if the block's key is a reconstructed CID (eq. CIDv1 with RAW IPLD codec) or multihash
+
 Datastore:
 
 #### `repo.datastore`

diff --git a/src/blockstore-utils.js b/src/blockstore-utils.js
@@ -7,23 +7,58 @@ const CID = require('cids')
 /**
  * Transform a cid to the appropriate datastore key.
  *
- * @param {CID} cid
+ * @param {Buffer} multihash
  * @returns {Key}
  */
-exports.cidToKey = cid => {
+exports.multihashToKey = multihash => {
   const enc = new base32.Encoder()
-  return new Key('/' + enc.write(cid.buffer).finalize(), false)
+  return new Key('/' + enc.write(multihash).finalize(), false)
 }
 
 /**
  * Transform a datastore Key instance to a CID
+ * As Key is a multihash of the CID, it is reconstructed using IPLD's RAW codec.
+ * Hence it is highly probable that stored CID will differ from a CID retrieved from blockstore.
  *
  * @param {Key} key
  * @returns {CID}
  */
-exports.keyToCid = key => {
+
+function keyToCid (key) {
+  // Block key is of the form /<base32 encoded string>
+  const decoder = new base32.Decoder()
+  const buff = decoder.write(key.toString().slice(1)).finalize()
+  return new CID(1, 'raw', Buffer.from(buff))
+}
+
+exports.keyToCid = keyToCid
+
+/**
+ * Transform a datastore Key instance to a multihash instance.
+ *
+ * @param {Key} key
+ * @returns {Buffer}
+ */
+
+function keyToMultihash (key) {
   // Block key is of the form /<base32 encoded string>
   const decoder = new base32.Decoder()
   const buff = decoder.write(key.toString().slice(1)).finalize()
-  return new CID(Buffer.from(buff))
+  return Buffer.from(buff)
+}
+
+exports.keyToMultihash = keyToMultihash
+
+/**
+ * Transforms a datastore Key containing multihash to a Key that contains reconstructed CID
+ *
+ * @param {Key} key
+ * @returns {CID}
+ */
+function keyToCidKey (key) {
+  const cid = keyToCid(key)
+  const enc = new base32.Encoder()
+  return new Key('/' + enc.write(cid.buffer).finalize(), false)
 }
+
+exports.keyToCidKey = keyToCidKey
diff --git a/src/blockstore.js b/src/blockstore.js
@@ -5,7 +5,7 @@ const ShardingStore = core.ShardingDatastore
 const Block = require('ipfs-block')
 const CID = require('cids')
 const errcode = require('err-code')
-const { cidToKey } = require('./blockstore-utils')
+const { multihashToKey, keyToCidKey } = require('./blockstore-utils')
 
 module.exports = async (filestore, options) => {
   const store = await maybeWithSharding(filestore, options)
@@ -26,10 +26,15 @@ function createBaseStore (store) {
      * Query the store.
      *
      * @param {object} query
+     * @param {boolean} reconstructsCids - Defines if Keys are converted to a reconstructed CID using IPLD_RAW codec
      * @return {Iterable}
      */
-    async * query (query) {
+    async * query (query, reconstructsCids = false) {
       for await (const block of store.query(query)) {
+        if (reconstructsCids) {
+          block.key = keyToCidKey(block.key)
+        }
+
         yield block
       }
     },
@@ -43,27 +48,9 @@ function createBaseStore (store) {
       if (!CID.isCID(cid)) {
         throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID')
       }
-      const key = cidToKey(cid)
-      let blockData
-      try {
-        blockData = await store.get(key)
-        return new Block(blockData, cid)
-      } catch (err) {
-        if (err.code === 'ERR_NOT_FOUND') {
-          const otherCid = cidToOtherVersion(cid)
-
-          if (!otherCid) {
-            throw err
-          }
-
-          const otherKey = cidToKey(otherCid)
-          const blockData = await store.get(otherKey)
-          await store.put(key, blockData)
-          return new Block(blockData, cid)
-        }
-
-        throw err
-      }
+      const key = multihashToKey(cid.multihash)
+      const blockData = await store.get(key)
+      return new Block(blockData, cid)
     },
     /**
      * Write a single block to the store.
@@ -76,7 +63,7 @@ function createBaseStore (store) {
         throw new Error('invalid block')
       }
 
-      const k = cidToKey(block.cid)
+      const k = multihashToKey(block.cid.multihash)
       const exists = await store.has(k)
       if (exists) return
       return store.put(k, block.data)
@@ -92,7 +79,7 @@ function createBaseStore (store) {
       const batch = store.batch()
 
       for await (const block of blocks) {
-        const key = cidToKey(block.cid)
+        const key = multihashToKey(block.cid.multihash)
 
         if (await store.has(key)) {
           continue
@@ -104,33 +91,38 @@ function createBaseStore (store) {
       return batch.commit()
     },
     /**
-     * Does the store contain block with this cid?
+     * Does the store contain block with this multihash or CID?
      *
-     * @param {CID} cid
-     * @returns {Promise<bool>}
+     * @param {CID|Buffer} obj
+     * @returns {Promise<boolean>}
      */
-    async has (cid) {
-      if (!CID.isCID(cid)) {
-        throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID')
+    has (obj) {
+      if (CID.isCID(obj)) {
+        obj = obj.multihash
       }
 
-      const exists = await store.has(cidToKey(cid))
-      if (exists) return exists
-      const otherCid = cidToOtherVersion(cid)
-      if (!otherCid) return false
-      return store.has(cidToKey(otherCid))
+      if (!Buffer.isBuffer(obj)) {
+        throw errcode(new Error('Not a valid key'), 'ERR_INVALID_KEY')
+      }
+
+      return store.has(multihashToKey(obj))
     },
     /**
-     * Delete a block from the store
+     * Delete a CID or multihash from the store
      *
-     * @param {CID} cid
+     * @param {CID|Buffer} obj
      * @returns {Promise<void>}
      */
-    async delete (cid) { // eslint-disable-line require-await
-      if (!CID.isCID(cid)) {
-        throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID')
+    async delete (obj) { // eslint-disable-line require-await
+      if (CID.isCID(obj)) {
+        obj = obj.multihash
+      }
+
+      if (!Buffer.isBuffer(obj)) {
+        throw errcode(new Error('Not a valid key'), 'ERR_INVALID_KEY')
       }
-      return store.delete(cidToKey(cid))
+
+      return store.delete(multihashToKey(obj))
     },
     /**
      * Close the store
@@ -142,11 +134,3 @@ function createBaseStore (store) {
     }
   }
 }
-
-function cidToOtherVersion (cid) {
-  try {
-    return cid.version === 0 ? cid.toV1() : cid.toV0()
-  } catch (err) {
-    return null
-  }
-}
diff --git a/src/constants.js b/src/constants.js
@@ -1,5 +1,5 @@
 'use strict'
 
 module.exports = {
-  repoVersion: 7
+  repoVersion: 8
 }
diff --git a/src/index.js b/src/index.js
@@ -277,11 +277,11 @@ class IpfsRepo {
     let count = new Big(0)
     let size = new Big(0)
 
-    for await (const block of this.blocks.query({})) {
+    for await (const block of this.blocks.query({}, false)) {
       count = count.plus(1)
       size = size
         .plus(block.value.byteLength)
-        .plus(block.key._buf.byteLength)
+        .plus(block.key.toBuffer().byteLength)
     }
 
     return { count, size }
@@ -292,7 +292,7 @@ async function getSize (queryFn) {
   let sum = new Big(0)
   for await (const block of queryFn.query({})) {
     sum.plus(block.value.byteLength)
-      .plus(block.key._buf.byteLength)
+      .plus(block.key.toBuffer().byteLength)
   }
   return sum
 }

diff --git a/test/blockstore-test.js b/test/blockstore-test.js
@@ -85,9 +85,11 @@ module.exports = (repo) => {
               close () {
 
               }
+
               has () {
                 return true
               }
+
               batch () {
                 return {
                   put () {
@@ -217,6 +219,7 @@ module.exports = (repo) => {
               close () {
 
               }
+
               get (c) {
                 if (c.toString() === key.toString()) {
                   throw err
@@ -278,7 +281,7 @@ module.exports = (repo) => {
           await repo.blocks.has('foo')
           throw new Error('Should have thrown')
         } catch (err) {
-          expect(err.code).to.equal('ERR_INVALID_CID')
+          expect(err.code).to.equal('ERR_INVALID_KEY')
         }
       })
 
@@ -304,7 +307,7 @@ module.exports = (repo) => {
           await repo.blocks.delete('foo')
           throw new Error('Should have thrown')
         } catch (err) {
-          expect(err.code).to.equal('ERR_INVALID_CID')
+          expect(err.code).to.equal('ERR_INVALID_KEY')
         }
       })
     })

diff --git a/test/blockstore-utils-test.js b/test/blockstore-utils-test.js
@@ -11,8 +11,9 @@ const Repo = require('../src')
 module.exports = () => {
   describe('blockstore utils', () => {
     it('converts a CID to a datastore Key and back', () => {
-      const originalCid = new CID('Qme6KJdKcp85TYbLxuLV7oQzMiLremD7HMoXLZEmgo6Rnh')
-      const key = Repo.utils.blockstore.cidToKey(originalCid)
+      // CIDv1 in base32 with IPLD raw codec
+      const originalCid = new CID('bafkreihkb3vrxxex5zvzkr3s3a6noe223r7jka4ofjy2nkzu27kueg76ii')
+      const key = Repo.utils.blockstore.multihashToKey(originalCid.multihash)
       expect(key instanceof Key).to.be.true()
       const cid = Repo.utils.blockstore.keyToCid(key)
       expect(cid instanceof CID).to.be.true()