From ff8824c45aeca33d2f53ae6babeb02f224e82a30 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Sat, 21 Jul 2018 23:26:59 +0100 Subject: [PATCH 1/5] fix: interoperability with go repo --- package.json | 3 ++- src/default-datastore.js | 35 ++++++++++++++++++++++++++++++++ src/index.js | 28 ++++++++++++++++++++++++- src/spec.js | 44 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 src/default-datastore.js create mode 100644 src/spec.js diff --git a/package.json b/package.json index 6397c3ff..3ee279e5 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,8 @@ "lodash.has": "^4.5.2", "lodash.set": "^4.3.2", "multiaddr": "^4.0.0", - "pull-stream": "^3.6.7" + "pull-stream": "^3.6.7", + "sort-keys": "^2.0.0" }, "license": "MIT", "contributors": [ diff --git a/src/default-datastore.js b/src/default-datastore.js new file mode 100644 index 00000000..68b63461 --- /dev/null +++ b/src/default-datastore.js @@ -0,0 +1,35 @@ +'use strict' + +// Default configuration for the datastore spec in node.js +module.exports = { + StorageMax: '10GB', + StorageGCWatermark: 90, + GCPeriod: '1h', + BloomFilterSize: 0, + Spec: { + type: 'mount', + mounts: [ + { + mountpoint: '/blocks', + type: 'measure', + prefix: 'flatfs.datastore', + child: { + type: 'flatfs', + path: 'blocks', + sync: true, + shardFunc: '/repo/flatfs/shard/v1/next-to-last/2' + } + }, + { + mountpoint: '/', + type: 'measure', + prefix: 'leveldb.datastore', + child: { + type: 'levelds', + path: 'datastore', + compression: 'none' + } + } + ] + } +} diff --git a/src/index.js b/src/index.js index 1e39f8b3..ed6e10fd 100644 --- a/src/index.js +++ b/src/index.js @@ -4,6 +4,7 @@ const waterfall = require('async/waterfall') const series = require('async/series') const parallel = require('async/parallel') const each = require('async/each') +const _get = require('lodash.get') const assert = require('assert') const path = require('path') const debug = require('debug') @@ -13,9 +14,11 @@ const pull = require('pull-stream') const backends = require('./backends') const version = require('./version') const config = require('./config') +const spec = require('./spec') const apiAddr = require('./api-addr') const blockstore = require('./blockstore') const defaultOptions = require('./default-options') +const defaultDatastore = require('./default-datastore') const ERRORS = require('./errors') const log = debug('repo') @@ -50,6 +53,7 @@ class IpfsRepo { this.root = backends.create('root', this.path, this.options) this.version = version(this.root) this.config = config(this.root) + this.spec = spec(this.root) this.apiAddr = apiAddr(this.root) } @@ -65,7 +69,8 @@ class IpfsRepo { series([ (cb) => this.root.open(ignoringAlreadyOpened(cb)), - (cb) => this.config.set(config, cb), + (cb) => this.config.set(buildConfig(config), cb), + (cb) => this.spec.set(buildDatastoreSpec(config), cb), (cb) => this.version.set(repoVersion, cb) ], callback) } @@ -209,6 +214,7 @@ class IpfsRepo { parallel( { config: (cb) => this.config.exists(cb), + spec: (cb) => this.spec.exists(cb), version: (cb) => this.version.check(repoVersion, cb) }, (err, res) => { @@ -377,3 +383,23 @@ function buildOptions (_options) { return options } + +function buildConfig (_config) { + _config.datastore = Object.assign({}, defaultDatastore, _get(_config, 'datastore', {})) + + return _config +} + +function buildDatastoreSpec (_config) { + const spec = Object.assign({}, defaultDatastore.Spec, _get(_config, 'datastore.Spec', {})) + + return { + type: spec.type, + mounts: spec.mounts.map((mounting) => ({ + mountpoint: mounting.mountpoint, + type: mounting.child.type, + path: mounting.child.path, + shardFunc: mounting.child.shardFunc + })) + } +} diff --git a/src/spec.js b/src/spec.js new file mode 100644 index 00000000..8cb165f9 --- /dev/null +++ b/src/spec.js @@ -0,0 +1,44 @@ +'use strict' + +const Key = require('interface-datastore').Key +const sortKeys = require('sort-keys') + +const specKey = new Key('datastore_spec') + +module.exports = (store) => { + return { + /** + * Check if a datastore spec file exists. + * + * @param {function(Error, bool)} callback + * @returns {void} + */ + exists (callback) { + store.has(specKey, callback) + }, + /** + * Get the current datastore spec. + * + * @param {function(Error, number)} callback + * @returns {void} + */ + get (callback) { + store.get(specKey, (err, buf) => { + if (err) { + return callback(err) + } + callback(null, JSON.parse(buf.toString())) + }) + }, + /** + * Set the datastore spec of the repo, writing it to the underlying store. + * + * @param {number} spec + * @param {function(Error)} callback + * @returns {void} + */ + set (spec, callback) { + store.put(specKey, Buffer.from(JSON.stringify(sortKeys(spec, { deep: true }))), callback) + } + } +} From ae74d33ac38c73844aa043cd50ca6a761460513a Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 23 Jul 2018 15:56:57 +0100 Subject: [PATCH 2/5] fix: interoperability with go repo - added tests --- src/constants.js | 2 +- src/default-datastore.js | 1 - test/repo-test.js | 24 ++++++++++++++++++++++-- test/test-repo/datastore_spec | 1 + test/test-repo/version | 2 +- 5 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 test/test-repo/datastore_spec diff --git a/src/constants.js b/src/constants.js index 91eb6f46..d680f22a 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,5 +1,5 @@ 'use strict' module.exports = { - repoVersion: 6 + repoVersion: 7 } diff --git a/src/default-datastore.js b/src/default-datastore.js index 68b63461..3b28a0df 100644 --- a/src/default-datastore.js +++ b/src/default-datastore.js @@ -2,7 +2,6 @@ // Default configuration for the datastore spec in node.js module.exports = { - StorageMax: '10GB', StorageGCWatermark: 90, GCPeriod: '1h', BloomFilterSize: 0, diff --git a/test/repo-test.js b/test/repo-test.js index 8d7c2d0f..2287cc00 100644 --- a/test/repo-test.js +++ b/test/repo-test.js @@ -61,11 +61,31 @@ module.exports = (repo) => { }) }) + describe('spec', () => { + it('get spec', (done) => { + repo.spec.get((err) => { + expect(err).to.not.exist() + done() + }) + }) + + it('set spec', (done) => { + series([ + (cb) => repo.spec.set({ a: 'b' }, cb), + (cb) => repo.spec.get((err, spec) => { + if (err) return cb(err) + expect(spec).to.deep.equal({ a: 'b' }) + cb() + }) + ], done) + }) + }) + describe('version', () => { it('get version', (done) => { repo.version.get((err, version) => { expect(err).to.not.exist() - expect(version).to.equal(6) + expect(version).to.equal(7) done() }) }) @@ -78,7 +98,7 @@ module.exports = (repo) => { expect(version).to.equal(9000) cb() }, - (cb) => repo.version.set(6, cb) + (cb) => repo.version.set(7, cb) ], done) }) }) diff --git a/test/test-repo/datastore_spec b/test/test-repo/datastore_spec new file mode 100644 index 00000000..7bf9626c --- /dev/null +++ b/test/test-repo/datastore_spec @@ -0,0 +1 @@ +{"mounts":[{"mountpoint":"/blocks","path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"} \ No newline at end of file diff --git a/test/test-repo/version b/test/test-repo/version index 1e8b3149..7f8f011e 100644 --- a/test/test-repo/version +++ b/test/test-repo/version @@ -1 +1 @@ -6 +7 From 01bf5704bc9565d13b319c12df5011ceb53721be Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 6 Aug 2018 09:34:27 +0100 Subject: [PATCH 3/5] chore: minor refactor --- src/default-datastore.js | 3 --- src/index.js | 1 + src/version.js | 4 ++++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/default-datastore.js b/src/default-datastore.js index 3b28a0df..b1043ddb 100644 --- a/src/default-datastore.js +++ b/src/default-datastore.js @@ -2,9 +2,6 @@ // Default configuration for the datastore spec in node.js module.exports = { - StorageGCWatermark: 90, - GCPeriod: '1h', - BloomFilterSize: 0, Spec: { type: 'mount', mounts: [ diff --git a/src/index.js b/src/index.js index ed6e10fd..21ef6c13 100644 --- a/src/index.js +++ b/src/index.js @@ -384,6 +384,7 @@ function buildOptions (_options) { return options } +// TODO this should come from js-ipfs instead function buildConfig (_config) { _config.datastore = Object.assign({}, defaultDatastore, _get(_config, 'datastore', {})) diff --git a/src/version.js b/src/version.js index 788f502e..fd499f4b 100644 --- a/src/version.js +++ b/src/version.js @@ -53,6 +53,10 @@ module.exports = (store) => { return callback(err) } log('comparing version: %s and %s', version, expected) + + // Version 6 and 7 are the same + expected = expected === 6 ? expected = 7 : expected + if (version !== expected) { return callback(new Error(`version mismatch: expected v${expected}, found v${version}`)) } From f67adf8ebcd478ec781c3cc199fb5df9e76b2914 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 23 Jul 2018 15:56:57 +0100 Subject: [PATCH 4/5] fix: interoperability with go repo - added tests --- src/default-datastore.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/default-datastore.js b/src/default-datastore.js index b1043ddb..d09fb1dc 100644 --- a/src/default-datastore.js +++ b/src/default-datastore.js @@ -2,6 +2,12 @@ // Default configuration for the datastore spec in node.js module.exports = { +<<<<<<< HEAD +======= + StorageGCWatermark: 90, + GCPeriod: '1h', + BloomFilterSize: 0, +>>>>>>> fix: interoperability with go repo - added tests Spec: { type: 'mount', mounts: [ From 7a75adc369ea01efafb2c1629a972826a07fb0e7 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 6 Aug 2018 09:34:27 +0100 Subject: [PATCH 5/5] chore: minor refactor --- src/default-datastore.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/default-datastore.js b/src/default-datastore.js index d09fb1dc..b1043ddb 100644 --- a/src/default-datastore.js +++ b/src/default-datastore.js @@ -2,12 +2,6 @@ // Default configuration for the datastore spec in node.js module.exports = { -<<<<<<< HEAD -======= - StorageGCWatermark: 90, - GCPeriod: '1h', - BloomFilterSize: 0, ->>>>>>> fix: interoperability with go repo - added tests Spec: { type: 'mount', mounts: [