WIP: fix: interoperability with go repo (#173)

* fix: interoperability with go repo

* fix: interoperability with go repo - added tests

* chore: minor refactor

* fix: interoperability with go repo - added tests

* chore: minor refactor

Author: vasco-santos

package.json

@@ -64,7 +64,8 @@
     "lodash.has": "^4.5.2",
     "lodash.set": "^4.3.2",
     "multiaddr": "^4.0.0",
-    "pull-stream": "^3.6.7"
+    "pull-stream": "^3.6.7",
+    "sort-keys": "^2.0.0"
   },
   "license": "MIT",
   "contributors": [

src/constants.js

@@ -1,5 +1,5 @@
 'use strict'
 
 module.exports = {
-  repoVersion: 6
+  repoVersion: 7
 }

src/default-datastore.js

@@ -0,0 +1,31 @@
+'use strict'
+
+// Default configuration for the datastore spec in node.js
+module.exports = {
+  Spec: {
+    type: 'mount',
+    mounts: [
+      {
+        mountpoint: '/blocks',
+        type: 'measure',
+        prefix: 'flatfs.datastore',
+        child: {
+          type: 'flatfs',
+          path: 'blocks',
+          sync: true,
+          shardFunc: '/repo/flatfs/shard/v1/next-to-last/2'
+        }
+      },
+      {
+        mountpoint: '/',
+        type: 'measure',
+        prefix: 'leveldb.datastore',
+        child: {
+          type: 'levelds',
+          path: 'datastore',
+          compression: 'none'
+        }
+      }
+    ]
+  }
+}

src/index.js

@@ -4,6 +4,7 @@ const waterfall = require('async/waterfall')
 const series = require('async/series')
 const parallel = require('async/parallel')
 const each = require('async/each')
+const _get = require('lodash.get')
 const assert = require('assert')
 const path = require('path')
 const debug = require('debug')
@@ -13,9 +14,11 @@ const pull = require('pull-stream')
 const backends = require('./backends')
 const version = require('./version')
 const config = require('./config')
+const spec = require('./spec')
 const apiAddr = require('./api-addr')
 const blockstore = require('./blockstore')
 const defaultOptions = require('./default-options')
+const defaultDatastore = require('./default-datastore')
 const ERRORS = require('./errors')
 
 const log = debug('repo')
@@ -50,6 +53,7 @@ class IpfsRepo {
     this.root = backends.create('root', this.path, this.options)
     this.version = version(this.root)
     this.config = config(this.root)
+    this.spec = spec(this.root)
     this.apiAddr = apiAddr(this.root)
   }
 
@@ -65,7 +69,8 @@ class IpfsRepo {
 
     series([
       (cb) => this.root.open(ignoringAlreadyOpened(cb)),
-      (cb) => this.config.set(config, cb),
+      (cb) => this.config.set(buildConfig(config), cb),
+      (cb) => this.spec.set(buildDatastoreSpec(config), cb),
       (cb) => this.version.set(repoVersion, cb)
     ], callback)
   }
@@ -209,6 +214,7 @@ class IpfsRepo {
     parallel(
       {
         config: (cb) => this.config.exists(cb),
+        spec: (cb) => this.spec.exists(cb),
         version: (cb) => this.version.check(repoVersion, cb)
       },
       (err, res) => {
@@ -377,3 +383,24 @@ function buildOptions (_options) {
 
   return options
 }
+
+// TODO this should come from js-ipfs instead
+function buildConfig (_config) {
+  _config.datastore = Object.assign({}, defaultDatastore, _get(_config, 'datastore', {}))
+
+  return _config
+}
+
+function buildDatastoreSpec (_config) {
+  const spec = Object.assign({}, defaultDatastore.Spec, _get(_config, 'datastore.Spec', {}))
+
+  return {
+    type: spec.type,
+    mounts: spec.mounts.map((mounting) => ({
+      mountpoint: mounting.mountpoint,
+      type: mounting.child.type,
+      path: mounting.child.path,
+      shardFunc: mounting.child.shardFunc
+    }))
+  }
+}

src/spec.js

@@ -0,0 +1,44 @@
+'use strict'
+
+const Key = require('interface-datastore').Key
+const sortKeys = require('sort-keys')
+
+const specKey = new Key('datastore_spec')
+
+module.exports = (store) => {
+  return {
+    /**
+     * Check if a datastore spec file exists.
+     *
+     * @param {function(Error, bool)} callback
+     * @returns {void}
+     */
+    exists (callback) {
+      store.has(specKey, callback)
+    },
+    /**
+     * Get the current datastore spec.
+     *
+     * @param {function(Error, number)} callback
+     * @returns {void}
+     */
+    get (callback) {
+      store.get(specKey, (err, buf) => {
+        if (err) {
+          return callback(err)
+        }
+        callback(null, JSON.parse(buf.toString()))
+      })
+    },
+    /**
+     * Set the datastore spec of the repo, writing it to the underlying store.
+     *
+     * @param {number} spec
+     * @param {function(Error)} callback
+     * @returns {void}
+     */
+    set (spec, callback) {
+      store.put(specKey, Buffer.from(JSON.stringify(sortKeys(spec, { deep: true }))), callback)
+    }
+  }
+}

src/version.js

@@ -53,6 +53,10 @@ module.exports = (store) => {
           return callback(err)
         }
         log('comparing version: %s and %s', version, expected)
+
+        // Version 6 and 7 are the same
+        expected = expected === 6 ? expected = 7 : expected
+
         if (version !== expected) {
           return callback(new Error(`version mismatch: expected v${expected}, found v${version}`))
         }

test/repo-test.js

@@ -61,11 +61,31 @@ module.exports = (repo) => {
       })
     })
 
+    describe('spec', () => {
+      it('get spec', (done) => {
+        repo.spec.get((err) => {
+          expect(err).to.not.exist()
+          done()
+        })
+      })
+
+      it('set spec', (done) => {
+        series([
+          (cb) => repo.spec.set({ a: 'b' }, cb),
+          (cb) => repo.spec.get((err, spec) => {
+            if (err) return cb(err)
+            expect(spec).to.deep.equal({ a: 'b' })
+            cb()
+          })
+        ], done)
+      })
+    })
+
     describe('version', () => {
       it('get version', (done) => {
         repo.version.get((err, version) => {
           expect(err).to.not.exist()
-          expect(version).to.equal(6)
+          expect(version).to.equal(7)
           done()
         })
       })
@@ -78,7 +98,7 @@ module.exports = (repo) => {
             expect(version).to.equal(9000)
             cb()
           },
-          (cb) => repo.version.set(6, cb)
+          (cb) => repo.version.set(7, cb)
         ], done)
       })
     })

test/test-repo/datastore_spec

@@ -0,0 +1 @@
+{"mounts":[{"mountpoint":"/blocks","path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"}

test/test-repo/version

@@ -1 +1 @@
-6
+7