From e944f0d9aa33da7a33f1d6815823fd6bbb3040af Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 2 Dec 2018 21:20:41 +0000 Subject: [PATCH 1/6] refactor: async iterators Uses async await and async iterators to implement the proposal here https://github.com/ipfs/interface-datastore/pull/25 License: MIT Signed-off-by: Alan Shaw --- .gitignore | 1 + package.json | 10 +-- src/index.js | 183 +++++++++++++++++++++------------------------ test/browser.js | 44 ++++++----- test/index.spec.js | 53 +++++-------- test/node.js | 56 ++++++-------- 6 files changed, 156 insertions(+), 191 deletions(-) diff --git a/.gitignore b/.gitignore index 5400d0f..eb82212 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ dist test/test-repo/datastore init-default datastore-test +.vscode \ No newline at end of file diff --git a/package.json b/package.json index 4f1a36c..f3eb07f 100644 --- a/package.json +++ b/package.json @@ -41,17 +41,15 @@ }, "homepage": "https://github.com/ipfs/js-datastore-level#readme", "dependencies": { - "datastore-core": "~0.6.0", + "datastore-core": "~0.7.0", "encoding-down": "^6.0.2", - "interface-datastore": "~0.6.0", + "interface-datastore": "github:ipfs/interface-datastore#refactor/async-iterators", "level-js": "github:timkuijsten/level.js#idbunwrapper", "leveldown": "^5.0.0", - "levelup": "^4.0.1", - "pull-stream": "^3.6.9" + "levelup": "^4.0.1" }, "devDependencies": { - "aegir": "^15.3.1", - "async": "^2.6.1", + "aegir": "^17.1.1", "chai": "^4.2.0", "cids": "~0.5.5", "dirty-chai": "^2.0.1", diff --git a/src/index.js b/src/index.js index 1a0e292..c94c407 100644 --- a/src/index.js +++ b/src/index.js @@ -3,14 +3,12 @@ /* :: import type {Callback, Batch, Query, QueryResult, QueryEntry} from 'interface-datastore' */ -const pull = require('pull-stream') const levelup = require('levelup') - -const asyncFilter = require('interface-datastore').utils.asyncFilter -const asyncSort = require('interface-datastore').utils.asyncSort -const Key = require('interface-datastore').Key -const Errors = require('interface-datastore').Errors +const { Key, Errors, utils } = require('interface-datastore') const encode = require('encoding-down') +const { promisify } = require('util') + +const { filter, map, take, sortAll } = utils /** * A datastore backed by leveldb. @@ -50,59 +48,53 @@ class LevelDatastore { ) } - open (callback /* : Callback */) /* : void */ { - this.db.open((err) => { - if (err) { - return callback(Errors.dbOpenFailedError(err)) - } - callback() - }) + async open () /* : Promise */ { + try { + await this.db.open() + } catch (err) { + throw Errors.dbOpenFailedError(err) + } } - put (key /* : Key */, value /* : Buffer */, callback /* : Callback */) /* : void */ { - this.db.put(key.toString(), value, (err) => { - if (err) { - return callback(Errors.dbWriteFailedError(err)) - } - callback() - }) + async put (key /* : Key */, value /* : Buffer */) /* : Promise */ { + try { + await this.db.put(key.toString(), value) + } catch (err) { + throw Errors.dbWriteFailedError(err) + } } - get (key /* : Key */, callback /* : Callback */) /* : void */ { - this.db.get(key.toString(), (err, data) => { - if (err) { - return callback(Errors.notFoundError(err)) - } - callback(null, data) - }) + async get (key /* : Key */) /* : Promise */ { + let data + try { + data = await this.db.get(key.toString()) + } catch (err) { + if (err.notFound) throw Errors.notFoundError(err) + throw Errors.dbWriteFailedError(err) + } + return data } - has (key /* : Key */, callback /* : Callback */) /* : void */ { - this.db.get(key.toString(), (err, res) => { - if (err) { - if (err.notFound) { - callback(null, false) - return - } - callback(err) - return - } - - callback(null, true) - }) + async has (key /* : Key */) /* : Promise */ { + try { + await this.db.get(key.toString()) + } catch (err) { + if (err.notFound) return false + throw err + } + return true } - delete (key /* : Key */, callback /* : Callback */) /* : void */ { - this.db.del(key.toString(), (err) => { - if (err) { - return callback(Errors.dbDeleteFailedError(err)) - } - callback() - }) + async delete (key /* : Key */) /* : Promise */ { + try { + await this.db.del(key.toString()) + } catch (err) { + throw Errors.dbDeleteFailedError(err) + } } - close (callback /* : Callback */) /* : void */ { - this.db.close(callback) + async close () /* : Promise */ { + return this.db.close() } batch () /* : Batch */ { @@ -121,8 +113,8 @@ class LevelDatastore { key: key.toString() }) }, - commit: (callback /* : Callback */) /* : void */ => { - this.db.batch(ops, callback) + commit: async () /* : Promise */ => { + return this.db.batch(ops) } } } @@ -133,70 +125,65 @@ class LevelDatastore { values = !q.keysOnly } - const iter = this.db.db.iterator({ - keys: true, - values: values, - keyAsBuffer: true - }) - - const rawStream = (end, cb) => { - if (end) { - return iter.end((err) => { - cb(err || end) - }) - } - - iter.next((err, key, value) => { - if (err) { - return cb(err) - } - - if (err == null && key == null && value == null) { - return iter.end((err) => { - cb(err || true) - }) - } - - const res /* : QueryEntry */ = { - key: new Key(key, false) - } - - if (values) { - res.value = Buffer.from(value) - } - - cb(null, res) + let it = levelIteratorToIterator( + this.db.db.iterator({ + keys: true, + values: values, + keyAsBuffer: true }) - } + ) - let tasks = [rawStream] - let filters = [] + it = map(it, ({ key, value }) => { + const res /* : QueryEntry */ = { key: new Key(key, false) } + if (values) { + res.value = Buffer.from(value) + } + return res + }) if (q.prefix != null) { - const prefix = q.prefix - filters.push((e, cb) => cb(null, e.key.toString().startsWith(prefix))) + it = filter(it, e => e.key.toString().startsWith(q.prefix)) } - if (q.filters != null) { - filters = filters.concat(q.filters) + if (Array.isArray(q.filters)) { + it = q.filters.reduce((it, f) => filter(it, f), it) } - tasks = tasks.concat(filters.map(f => asyncFilter(f))) - - if (q.orders != null) { - tasks = tasks.concat(q.orders.map(o => asyncSort(o))) + if (Array.isArray(q.orders)) { + it = q.orders.reduce((it, f) => sortAll(it, f), it) } if (q.offset != null) { let i = 0 - tasks.push(pull.filter(() => i++ >= q.offset)) + it = filter(it, () => i++ >= q.offset) } if (q.limit != null) { - tasks.push(pull.take(q.limit)) + it = take(it, q.limit) } - return pull.apply(null, tasks) + return it + } +} + +function levelIteratorToIterator (li) { + return { + next: () => new Promise((resolve, reject) => { + li.next((err, key, value) => { + if (err) return reject(err) + if (key == null) return resolve({ done: true }) + resolve({ done: false, value: { key, value } }) + }) + }), + return: () => new Promise((resolve, reject) => { + li.end(err => { + if (err) return reject(err) + resolve({ done: true }) + }) + }), + [Symbol.asyncIterator] () { + return this + } } } diff --git a/test/browser.js b/test/browser.js index c049d3f..586d382 100644 --- a/test/browser.js +++ b/test/browser.js @@ -2,9 +2,8 @@ /* eslint-env mocha */ 'use strict' -const each = require('async/each') -const MountStore = require('datastore-core').MountDatastore -const Key = require('interface-datastore').Key +const { MountDatastore } = require('datastore-core') +const { Key } = require('interface-datastore') // leveldown will be swapped for level-js const leveljs = require('leveldown') @@ -14,31 +13,40 @@ const LevelStore = require('../src') describe('LevelDatastore', () => { describe('interface-datastore (leveljs)', () => { require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new LevelStore('hello', {db: leveljs})) - }, - teardown (callback) { - leveljs.destroy('hello', callback) - } + setup: () => new LevelStore('hello', { db: leveljs }), + teardown: () => new Promise((resolve, reject) => { + leveljs.destroy('hello', err => { + if (err) return reject(err) + resolve() + }) + }) }) }) - describe('interface-datastore (mount(leveljs, leveljs, leveljs))', () => { + // TODO: unskip when datastore-core is converted to async/await/iterators + describe.skip('interface-datastore (mount(leveljs, leveljs, leveljs))', () => { require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new MountStore([{ + setup () { + return new MountDatastore([{ prefix: new Key('/a'), - datastore: new LevelStore('one', {db: leveljs}) + datastore: new LevelStore('one', { db: leveljs }) }, { prefix: new Key('/q'), - datastore: new LevelStore('two', {db: leveljs}) + datastore: new LevelStore('two', { db: leveljs }) }, { prefix: new Key('/z'), - datastore: new LevelStore('three', {db: leveljs}) - }])) + datastore: new LevelStore('three', { db: leveljs }) + }]) }, - teardown (callback) { - each(['one', 'two', 'three'], leveljs.destroy.bind(leveljs), callback) + teardown () { + return Promise.all(['one', 'two', 'three'].map(dir => { + return new Promise((resolve, reject) => { + leveljs.destroy(dir, err => { + if (err) return reject(err) + resolve() + }) + }) + })) } }) }) diff --git a/test/index.spec.js b/test/index.spec.js index fdd62ef..11d9e16 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -7,64 +7,49 @@ chai.use(require('dirty-chai')) const expect = chai.expect const memdown = require('memdown') const LevelDown = require('leveldown') -const eachSeries = require('async/eachSeries') - const LevelStore = require('../src') describe('LevelDatastore', () => { describe('initialization', () => { - it('should default to a leveldown database', (done) => { + it('should default to a leveldown database', async () => { const levelStore = new LevelStore('init-default') + await levelStore.open() - levelStore.open((err) => { - expect(err).to.not.exist() - expect(levelStore.db.db.db instanceof LevelDown).to.equal(true) - expect(levelStore.db.options).to.include({ - createIfMissing: true, - errorIfExists: false - }) - expect(levelStore.db.db.codec.opts).to.include({ - valueEncoding: 'binary' - }) - done() + expect(levelStore.db.db.db instanceof LevelDown).to.equal(true) + expect(levelStore.db.options).to.include({ + createIfMissing: true, + errorIfExists: false + }) + expect(levelStore.db.db.codec.opts).to.include({ + valueEncoding: 'binary' }) }) - it('should be able to override the database', (done) => { + it('should be able to override the database', async () => { const levelStore = new LevelStore('init-default', { db: memdown, createIfMissing: true, errorIfExists: true }) - levelStore.open((err) => { - expect(err).to.not.exist() - expect(levelStore.db.db.db instanceof memdown).to.equal(true) - expect(levelStore.db.options).to.include({ - createIfMissing: true, - errorIfExists: true - }) - done() + await levelStore.open() + + expect(levelStore.db.db.db instanceof memdown).to.equal(true) + expect(levelStore.db.options).to.include({ + createIfMissing: true, + errorIfExists: true }) }) }) - eachSeries([ - memdown, - LevelDown - ], (database) => { + ;[memdown, LevelDown].forEach(database => { describe(`interface-datastore ${database.name}`, () => { require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new LevelStore('datastore-test', {db: database})) - }, - teardown (callback) { + setup: () => new LevelStore(`datastore-test/${Math.random()}`, { db: database }), + teardown () { memdown.clearGlobalStore() - callback() } }) }) - }, (err) => { - expect(err).to.not.exist() }) }) diff --git a/test/node.js b/test/node.js index b6f8b02..5ef1cc7 100644 --- a/test/node.js +++ b/test/node.js @@ -5,14 +5,12 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream') const path = require('path') -const utils = require('interface-datastore').utils +const { Key, utils } = require('interface-datastore') const rimraf = require('rimraf') -const each = require('async/each') -const MountStore = require('datastore-core').MountDatastore -const Key = require('interface-datastore').Key +const { MountDatastore } = require('datastore-core') const CID = require('cids') +const { promisify } = require('util') const LevelStore = require('../src') @@ -20,18 +18,13 @@ describe('LevelDatastore', () => { describe('interface-datastore (leveldown)', () => { const dir = utils.tmpdir() require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new LevelStore(dir, { - db: require('leveldown') - })) - }, - teardown (callback) { - rimraf(dir, callback) - } + setup: () => new LevelStore(dir, { db: require('leveldown') }), + teardown: () => promisify(rimraf)(dir) }) }) - describe('interface-datastore (mount(leveldown, leveldown, leveldown))', () => { + // TODO: unskip when datastore-core is converted to async/await/iterators + describe.skip('interface-datastore (mount(leveldown, leveldown, leveldown))', () => { const dirs = [ utils.tmpdir(), utils.tmpdir(), @@ -39,8 +32,8 @@ describe('LevelDatastore', () => { ] require('interface-datastore/src/tests')({ - setup (callback) { - callback(null, new MountStore([{ + setup () { + return new MountDatastore([{ prefix: new Key('/a'), datastore: new LevelStore(dirs[0], { db: require('leveldown') @@ -55,33 +48,26 @@ describe('LevelDatastore', () => { datastore: new LevelStore(dirs[2], { db: require('leveldown') }) - }])) + }]) }, - teardown (callback) { - each(dirs, rimraf, callback) + teardown () { + return Promise.all(dirs.map(dir => promisify(rimraf)(dir))) } }) }) - it.skip('interop with go', (done) => { + it.skip('interop with go', async () => { const store = new LevelStore(path.join(__dirname, 'test-repo', 'datastore'), { db: require('leveldown') }) - pull( - store.query({}), - pull.map((e) => { - // console.log('=======') - // console.log(e) - // console.log(e.key.toBuffer().toString()) - return new CID(1, 'dag-cbor', e.key.toBuffer()) - }), - pull.collect((err, cids) => { - expect(err).to.not.exist() - expect(cids[0].version).to.be.eql(0) - expect(cids).to.have.length(4) - done() - }) - ) + let cids = [] + + for await (const e of store.query({})) { + cids.push(new CID(1, 'dag-cbor', e.key.toBuffer())) + } + + expect(cids[0].version).to.be.eql(0) + expect(cids).to.have.length(4) }) }) From 74d4a36d1bca963f6953614e3901868818ae55da Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 2 Dec 2018 21:37:13 +0000 Subject: [PATCH 2/6] fix: remove unused var License: MIT Signed-off-by: Alan Shaw --- src/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/index.js b/src/index.js index c94c407..d0a0b1e 100644 --- a/src/index.js +++ b/src/index.js @@ -6,7 +6,6 @@ const levelup = require('levelup') const { Key, Errors, utils } = require('interface-datastore') const encode = require('encoding-down') -const { promisify } = require('util') const { filter, map, take, sortAll } = utils From 601599d95cde884c3fd1ad4d5ff88929de65c55c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 2 Dec 2018 21:50:36 +0000 Subject: [PATCH 3/6] fix: tests License: MIT Signed-off-by: Alan Shaw --- test/index.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/index.spec.js b/test/index.spec.js index 11d9e16..e7c7d94 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -7,6 +7,7 @@ chai.use(require('dirty-chai')) const expect = chai.expect const memdown = require('memdown') const LevelDown = require('leveldown') +const os = require('os') const LevelStore = require('../src') describe('LevelDatastore', () => { @@ -45,7 +46,7 @@ describe('LevelDatastore', () => { ;[memdown, LevelDown].forEach(database => { describe(`interface-datastore ${database.name}`, () => { require('interface-datastore/src/tests')({ - setup: () => new LevelStore(`datastore-test/${Math.random()}`, { db: database }), + setup: () => new LevelStore(`${os.tmpdir()}/datastore-level-test-${Math.random()}`, { db: database }), teardown () { memdown.clearGlobalStore() } From a01eb377b24268cf92af8ab826ebccf222e4afa0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 29 May 2019 14:19:31 +0100 Subject: [PATCH 4/6] chore: update deps and unskip tests --- package.json | 4 ++-- test/browser.js | 3 +-- test/node.js | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index f3eb07f..4f707d2 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "homepage": "https://github.com/ipfs/js-datastore-level#readme", "dependencies": { "datastore-core": "~0.7.0", - "encoding-down": "^6.0.2", - "interface-datastore": "github:ipfs/interface-datastore#refactor/async-iterators", + "encoding-down": "^6.0.4", + "interface-datastore": "~0.7.0", "level-js": "github:timkuijsten/level.js#idbunwrapper", "leveldown": "^5.0.0", "levelup": "^4.0.1" diff --git a/test/browser.js b/test/browser.js index 586d382..18cdaa6 100644 --- a/test/browser.js +++ b/test/browser.js @@ -23,8 +23,7 @@ describe('LevelDatastore', () => { }) }) - // TODO: unskip when datastore-core is converted to async/await/iterators - describe.skip('interface-datastore (mount(leveljs, leveljs, leveljs))', () => { + describe('interface-datastore (mount(leveljs, leveljs, leveljs))', () => { require('interface-datastore/src/tests')({ setup () { return new MountDatastore([{ diff --git a/test/node.js b/test/node.js index 5ef1cc7..c15ae37 100644 --- a/test/node.js +++ b/test/node.js @@ -23,8 +23,7 @@ describe('LevelDatastore', () => { }) }) - // TODO: unskip when datastore-core is converted to async/await/iterators - describe.skip('interface-datastore (mount(leveldown, leveldown, leveldown))', () => { + describe('interface-datastore (mount(leveldown, leveldown, leveldown))', () => { const dirs = [ utils.tmpdir(), utils.tmpdir(), From 7c1a51d74e976e6c373808ce62dd0cbd74d25d93 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 29 May 2019 14:45:58 +0100 Subject: [PATCH 5/6] chore: update deps --- package.json | 12 ++++++------ test/index.spec.js | 4 +--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 4f707d2..3a630cd 100644 --- a/package.json +++ b/package.json @@ -42,19 +42,19 @@ "homepage": "https://github.com/ipfs/js-datastore-level#readme", "dependencies": { "datastore-core": "~0.7.0", - "encoding-down": "^6.0.4", + "encoding-down": "^6.0.2", "interface-datastore": "~0.7.0", - "level-js": "github:timkuijsten/level.js#idbunwrapper", "leveldown": "^5.0.0", "levelup": "^4.0.1" }, "devDependencies": { - "aegir": "^17.1.1", + "aegir": "^19.0.3", "chai": "^4.2.0", - "cids": "~0.5.5", + "cids": "~0.7.1", "dirty-chai": "^2.0.1", - "flow-bin": "~0.81.0", - "memdown": "^1.4.1", + "flow-bin": "~0.99.0", + "level-js": "^4.0.1", + "memdown": "^4.0.0", "rimraf": "^2.6.2" }, "contributors": [ diff --git a/test/index.spec.js b/test/index.spec.js index e7c7d94..0bf09d1 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -47,9 +47,7 @@ describe('LevelDatastore', () => { describe(`interface-datastore ${database.name}`, () => { require('interface-datastore/src/tests')({ setup: () => new LevelStore(`${os.tmpdir()}/datastore-level-test-${Math.random()}`, { db: database }), - teardown () { - memdown.clearGlobalStore() - } + teardown () {} }) }) }) From 3777399739161fe55553e35c6a793b05342908d0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 29 May 2019 14:52:41 +0100 Subject: [PATCH 6/6] chore: fix linting --- src/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index d0a0b1e..87188db 100644 --- a/src/index.js +++ b/src/index.js @@ -92,7 +92,7 @@ class LevelDatastore { } } - async close () /* : Promise */ { + close () /* : Promise */ { return this.db.close() } @@ -112,7 +112,7 @@ class LevelDatastore { key: key.toString() }) }, - commit: async () /* : Promise */ => { + commit: () /* : Promise */ => { return this.db.batch(ops) } }