diff --git a/index.js b/index.js index b8cc713..14bf2f4 100644 --- a/index.js +++ b/index.js @@ -13,8 +13,8 @@ var bulk = require('bulk-write-stream') var events = require('events') var sodium = require('sodium-universal') var alru = require('array-lru') +var pathBuilder = require('./lib/path') var inherits = require('inherits') -var hash = require('./lib/hash') var iterator = require('./lib/iterator') var differ = require('./lib/differ') var history = require('./lib/history') @@ -56,6 +56,7 @@ function HyperDB (storage, key, opts) { this.id = Buffer.alloc(32) sodium.randombytes_buf(this.id) + this._path = pathBuilder(opts) this._storage = createStorage(storage) this._contentStorage = typeof opts.contentFeed === 'function' ? opts.contentFeed @@ -106,7 +107,7 @@ HyperDB.prototype.batch = function (batch, cb) { if (err) return done(err) if (node) { - node.path = hash(node.key, true) + node.path = self._path(node.key, true) heads = [node] } @@ -756,7 +757,7 @@ Writer.prototype._decode = function (seq, buf, cb) { var val = messages.Entry.decode(buf) val[util.inspect.custom] = inspect val.seq = seq - val.path = hash(val.key, true) + val.path = this._db._path(val.key, true) val.value = val.value && this._db._valueEncoding.decode(val.value) if (this._feedsMessage && this._feedsLoaded === val.inflate) { diff --git a/lib/get.js b/lib/get.js index d796afd..82ce3a8 100644 --- a/lib/get.js +++ b/lib/get.js @@ -1,4 +1,3 @@ -var hash = require('./hash') var options = require('./options') module.exports = get @@ -17,7 +16,7 @@ function GetRequest (db, key, opts) { this._callback = noop this._options = opts || null this._prefixed = !!(opts && opts.prefix) - this._path = (opts && opts.path) || hash(key, !this._prefixed) + this._path = (opts && opts.path) || db._path(key, !this._prefixed) this._db = db this._error = null this._active = 0 diff --git a/lib/hash.js b/lib/hash.js deleted file mode 100644 index 1feaabd..0000000 --- a/lib/hash.js +++ /dev/null @@ -1,44 +0,0 @@ -var sodium = require('sodium-universal') - -var KEY = Buffer.alloc(16) -var OUT = Buffer.alloc(8) - -hash.TERMINATE = 4 -hash.LENGTH = 32 - -module.exports = hash - -function hash (keys, terminate) { - if (typeof keys === 'string') keys = split(keys) - - var all = new Array(keys.length * 32 + (terminate ? 1 : 0)) - - for (var i = 0; i < keys.length; i++) { - sodium.crypto_shorthash(OUT, Buffer.from(keys[i]), KEY) - expandHash(OUT, all, i * 32) - } - - if (terminate) all[all.length - 1] = 4 - - return all -} - -function expandHash (next, out, offset) { - for (var i = 0; i < next.length; i++) { - var n = next[i] - - for (var j = 0; j < 4; j++) { - var r = n & 3 - out[offset++] = r - n -= r - n /= 4 - } - } -} - -function split (key) { - var list = key.split('/') - if (list[0] === '') list.shift() - if (list[list.length - 1] === '') list.pop() - return list -} diff --git a/lib/iterator.js b/lib/iterator.js index bf8e272..7d2e829 100644 --- a/lib/iterator.js +++ b/lib/iterator.js @@ -1,10 +1,10 @@ var nanoiterator = require('nanoiterator') var inherits = require('inherits') -var hash = require('./hash') +var path = require('./path') var options = require('./options') -var SORT_GT = [3, 2, 1, 0] -var SORT_GTE = [3, 2, 1, 0, 4] +var SORT_GT = [3, 2, 1, 0, path.SEPARATE] +var SORT_GTE = [3, 2, 1, 0, path.SEPARATE, path.TERMINATE] module.exports = Iterator @@ -16,7 +16,7 @@ function Iterator (db, prefix, opts) { this._db = db this._stack = [{ - path: prefix ? hash(prefix, false) : [], + path: prefix ? this._db._path(prefix, false) : [], node: null, i: 0 }] @@ -24,7 +24,6 @@ function Iterator (db, prefix, opts) { this._recursive = opts.recursive !== false this._gt = !!opts.gt this._start = this._stack[0].path.length - this._end = this._recursive ? Infinity : this._start + hash.LENGTH this._map = options.map(opts, db) this._reduce = options.reduce(opts, db) this._collisions = [] @@ -72,9 +71,10 @@ Iterator.prototype._pushPrefix = function (path, i, val) { // fast case Iterator.prototype._singleNode = function (top, cb) { var node = top.node - var end = Math.min(this._end, node.trie.length) - for (var i = top.i; i < end; i++) { + for (var i = top.i; i < node.trie.length; i++) { + if (!this._recursive && node.path[i - 1] === path.SEPARATE) break + var bucket = i < node.trie.length && node.trie[i] if (!bucket) continue @@ -114,14 +114,12 @@ Iterator.prototype._multiNode = function (path, nodes, cb) { var ptr = path.length - if (ptr < this._end) { - var order = this._sortOrder(ptr) + var order = this._sortOrder(ptr) - for (var i = 0; i < order.length; i++) { - var sortValue = order[i] - if (!visitTrie(nodes, ptr, sortValue)) continue - this._pushPrefix(path, path.length, sortValue) - } + for (var i = 0; i < order.length; i++) { + var sortValue = order[i] + if (!visitTrie(nodes, ptr, sortValue)) continue + this._pushPrefix(path, ptr, sortValue) } nodes = this._filterResult(nodes, ptr) @@ -132,11 +130,10 @@ Iterator.prototype._multiNode = function (path, nodes, cb) { Iterator.prototype._filterResult = function (nodes, i) { var result = null - nodes.sort(byKey) - for (var j = 0; j < nodes.length; j++) { var node = nodes[j] - if (node.path.length !== i && i !== this._end) continue + if (this._recursive && node.path.length !== i) continue + if (!this._recursive && node.path[i - 1] !== path.SEPARATE) continue if (!isPrefix(node.key, this._prefix)) continue if (!result) result = [] @@ -190,11 +187,6 @@ Iterator.prototype._sortOrder = function (i) { return gt && this._start === i ? SORT_GT : SORT_GTE } -function byKey (a, b) { - var k = b.key.localeCompare(a.key) - return k || b.feed - a.feed -} - function allDeletes (nodes) { for (var i = 0; i < nodes.length; i++) { if (nodes[i].value) return false diff --git a/lib/path.js b/lib/path.js new file mode 100644 index 0000000..d796287 --- /dev/null +++ b/lib/path.js @@ -0,0 +1,77 @@ +var sodium = require('sodium-universal') + +var KEY = Buffer.alloc(16) +var OUT = Buffer.alloc(8) + +path.TERMINATE = 5 +path.SEPARATE = 4 + +function path (opts) { + if (!opts) return lexint + return opts.lexint ? lexint : hash +} + +module.exports = path + +function lexint (keys, terminate) { + if (typeof keys === 'string') keys = split(keys) + if (!keys.length) return [] + + var lengths = keys.map(k => k.length) + var totalSize = lengths.reduce(sum, 0) * 4 + + var all = new Array(totalSize + keys.length + (terminate ? 1 : 0)) + + var offset = 0 + for (var i = 0; i < keys.length; i++) { + var buf = Buffer.from(keys[i]) + expandComponent(buf, all, offset) + offset += lengths[i] * 4 + all[offset++] = path.SEPARATE + } + + if (terminate) all[all.length - 1] = path.TERMINATE + + return all +} + +function hash (keys, terminate) { + if (typeof keys === 'string') keys = split(keys) + if (!keys.length) return [] + + var all = new Array(keys.length * 32 + keys.length + (terminate ? 1 : 0)) + + var offset = 0 + for (var i = 0; i < keys.length; i++) { + sodium.crypto_shorthash(OUT, Buffer.from(keys[i]), KEY) + expandComponent(OUT, all, offset) + offset += 32 + all[offset++] = path.SEPARATE + } + + if (terminate) all[all.length - 1] = path.TERMINATE + + return all +} + +function expandComponent (next, out, offset) { + for (var i = 0; i < next.length; i++) { + var n = next[i] + + for (var j = 3; j >= 0; j--) { + var r = n & 3 + out[offset + 4 * i + j] = r + n -= r + n /= 4 + } + } +} + +function sum (s, n) { return s + n } + +function split (key) { + var list = key.split('/') + if (list[0] === '') list.shift() + if (list[list.length - 1] === '') list.pop() + return list +} diff --git a/lib/put.js b/lib/put.js index 6c30b90..2b4446a 100644 --- a/lib/put.js +++ b/lib/put.js @@ -1,4 +1,4 @@ -var hash = require('./hash') +var path = require('./path') module.exports = put @@ -16,7 +16,7 @@ function PutRequest (db, key, value, clock) { this._error = null this._callback = noop this._db = db - this._path = hash(key, true) + this._path = db._path(key, true) this._trie = [] } @@ -107,13 +107,13 @@ PutRequest.prototype._copyTrie = function (worker, bucket, val) { // check if we are the closest node, if so skip this // except if we are terminating the val. if so we // need to check for collions before making the decision - if (i === val && val !== 4) continue + if (i === val && val !== path.TERMINATE) continue var ptrs = bucket[i] || [] for (var k = 0; k < ptrs.length; k++) { var ptr = ptrs[k] // if termination value, push if get(ptr).key !== key - if (val === 4) this._checkCollision(worker, i, ptr.feed, ptr.seq) + if (val === path.TERMINATE) this._checkCollision(worker, i, ptr.feed, ptr.seq) else this._push(worker, i, ptr.feed, ptr.seq) } } @@ -125,7 +125,7 @@ PutRequest.prototype._splitTrie = function (worker, bucket, val) { // check if we need to split the trie at all // i.e. is head still closest and is head not a conflict - if (headVal === val && (headVal < 4 || head.key === this.key)) return + if (headVal === val && (headVal < path.TERMINATE || head.key === this.key)) return // push head to the trie this._push(worker, headVal, head.feed, head.seq) diff --git a/test/helpers/create.js b/test/helpers/create.js index 9839d7c..11a4a49 100644 --- a/test/helpers/create.js +++ b/test/helpers/create.js @@ -6,22 +6,32 @@ var reduce = (a, b) => a exports.one = function (key, opts) { if (!opts) opts = {} - opts.reduce = reduce - opts.valueEncoding = opts.valueEncoding || 'utf-8' - var storage = opts.latency ? name => latency(opts.latency, ram()) : ram - return hyperdb(storage, key, opts) + var options = Object.assign({ + reduce, + valueEncoding: 'utf-8' + }, opts) + var storage = options.latency ? name => latency(options.latency, ram()) : ram + return hyperdb(storage, key, options) } -exports.two = function (cb) { - createMany(2, function (err, dbs, replicateByIndex) { +exports.two = function (opts, cb) { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + createMany(2, opts, function (err, dbs, replicateByIndex) { if (err) return cb(err) dbs.push(replicateByIndex.bind(null, [0, 1])) return cb.apply(null, dbs) }) } -exports.three = function (cb) { - createMany(3, function (err, dbs, replicateByIndex) { +exports.three = function (opts, cb) { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + createMany(3, opts, function (err, dbs, replicateByIndex) { if (err) return cb(err) dbs.push(replicateByIndex.bind(null, [0, 1, 2])) return cb.apply(null, dbs) @@ -30,11 +40,15 @@ exports.three = function (cb) { exports.many = createMany -function createMany (count, cb) { +function createMany (count, opts, cb) { + if (typeof opts === 'function') return createMany(count, {}, opts) + var dbs = [] var remaining = count - 1 - var first = hyperdb(ram, { valueEncoding: 'utf-8' }) + var options = Object.assign({}, { valueEncoding: 'utf-8' }, opts) + + var first = hyperdb(ram, options) first.ready(function (err) { if (err) return cb(err) dbs.push(first) @@ -49,7 +63,7 @@ function createMany (count, cb) { return cb(null, dbs, replicateByIndex) }) } - var db = hyperdb(ram, first.key, { valueEncoding: 'utf-8' }) + var db = hyperdb(ram, first.key, options) db.ready(function (err) { if (err) return cb(err) first.authorize(db.local.key, function (err) { diff --git a/test/iterator-order.js b/test/iterator-order.js index cbbef4b..5f67318 100644 --- a/test/iterator-order.js +++ b/test/iterator-order.js @@ -2,12 +2,15 @@ var tape = require('tape') var create = require('./helpers/create') var put = require('./helpers/put') var run = require('./helpers/run') -var hash = require('../lib/hash') +var pathBuilder = require('../lib/path') -function sortByHash (a, b) { - var ha = hash(typeof a === 'string' ? a : a.key).join('') - var hb = hash(typeof b === 'string' ? b : b.key).join('') - return ha.localeCompare(hb) +function getSortFunction (opts) { + var path = pathBuilder(opts) + return function (a, b) { + var ha = path(typeof a === 'string' ? a : a.key).join('') + var hb = path(typeof b === 'string' ? b : b.key).join('') + return ha.localeCompare(hb) + } } const cases = { @@ -16,70 +19,89 @@ const cases = { '3 paths deep': ['a', 'a/a', 'a/b', 'a/c', 'a/a/a', 'a/a/b', 'a/a/c'] } -Object.keys(cases).forEach((key) => { - tape('iterator is hash order sorted (' + key + ')', function (t) { - var keysToTest = cases[key] - run( - cb => testSingleFeedWithKeys(t, keysToTest, cb), - cb => testTwoFeedsWithKeys(t, keysToTest, cb), - cb => t.end() - ) - }) -}) +runIterationOrderSuite({ lexint: false }) +runIterationOrderSuite({ lexint: true }) -tape('fully visit a folder before visiting the next one', function (t) { - t.plan(12) - var db = create.one() - put(db, ['a', 'a/b', 'a/b/c', 'b/c', 'b/c/d'], function (err) { - t.error(err, 'no error') - var ite = db.iterator() +function runIterationOrderSuite (opts) { + run( + cb => fullyVisitFolder(opts, cb), + cb => testAllCases(opts, cb) + ) +} - ite.next(function loop (err, val) { +function testAllCases (opts, cb) { + var sorter = getSortFunction(opts) + var tag = opts.lexint ? 'lexint' : 'hash' + Object.keys(cases).forEach((key) => { + tape('iterator is ' + tag + ' order sorted (' + key + ')', function (t) { + var keysToTest = cases[key] + run( + cb => testSingleFeedWithKeys(t, opts, sorter, keysToTest, cb), + cb => testTwoFeedsWithKeys(t, opts, sorter, keysToTest, cb), + cb => t.end() + ) + }) + }) +} + +function fullyVisitFolder (opts, cb) { + tape('fully visit a folder before visiting the next one', function (t) { + t.plan(12) + var db = create.one(null, opts) + put(db, ['a', 'a/b', 'a/b/c', 'b/c', 'b/c/d'], function (err) { t.error(err, 'no error') - if (!val) return t.end() + var ite = db.iterator() - if (val.key[0] === 'b') { - t.same(val.key, 'b/c') - ite.next(function (err, val) { - t.error(err, 'no error') - t.same(val.key, 'b/c/d') - ite.next(loop) - }) - } else { - t.same(val.key, 'a') - ite.next(function (err, val) { - t.error(err, 'no error') - t.same(val.key, 'a/b') + ite.next(function loop (err, val) { + t.error(err, 'no error') + if (!val) { + t.end() + return cb() + } + + if (val.key[0] === 'b') { + t.same(val.key, 'b/c') ite.next(function (err, val) { t.error(err, 'no error') - t.same(val.key, 'a/b/c') + t.same(val.key, 'b/c/d') ite.next(loop) }) - }) - } + } else { + t.same(val.key, 'a') + ite.next(function (err, val) { + t.error(err, 'no error') + t.same(val.key, 'a/b') + ite.next(function (err, val) { + t.error(err, 'no error') + t.same(val.key, 'a/b/c') + ite.next(loop) + }) + }) + } + }) }) }) -}) +} -function testSingleFeedWithKeys (t, keys, cb) { +function testSingleFeedWithKeys (t, opts, sorter, keys, cb) { t.comment('with single feed') - var db = create.one() + var db = create.one(null, opts) put(db, keys, function (err) { t.error(err, 'no error') - testIteratorOrder(t, db.iterator(), keys, cb) + testIteratorOrder(t, sorter, db.iterator(), keys, cb) }) } -function testTwoFeedsWithKeys (t, keys, cb) { +function testTwoFeedsWithKeys (t, opts, sorter, keys, cb) { t.comment('with values split across two feeds') - create.two(function (db1, db2, replicate) { + create.two(opts, function (db1, db2, replicate) { var half = Math.floor(keys.length / 2) run( cb => put(db1, keys.slice(0, half), cb), cb => put(db2, keys.slice(half), cb), cb => replicate(cb), - cb => testIteratorOrder(t, db1.iterator(), keys, cb), - cb => testIteratorOrder(t, db2.iterator(), keys, cb), + cb => testIteratorOrder(t, sorter, db1.iterator(), keys, cb), + cb => testIteratorOrder(t, sorter, db2.iterator(), keys, cb), done ) }) @@ -89,8 +111,8 @@ function testTwoFeedsWithKeys (t, keys, cb) { } } -function testIteratorOrder (t, iterator, expected, done) { - var sorted = expected.slice(0).sort(sortByHash) +function testIteratorOrder (t, sorter, iterator, expected, done) { + var sorted = expected.slice(0).sort(sorter) each(iterator, onEach, onDone) function onEach (err, node) { t.error(err, 'no error') diff --git a/test/iterator.js b/test/iterator.js index 45da4aa..45c1084 100644 --- a/test/iterator.js +++ b/test/iterator.js @@ -3,200 +3,190 @@ var create = require('./helpers/create') var put = require('./helpers/put') var run = require('./helpers/run') -tape('basic iteration', function (t) { - var db = create.one() - var vals = ['a', 'b', 'c'] - var expected = toMap(vals) - - put(db, vals, function (err) { - t.error(err, 'no error') - all(db.iterator(), function (err, map) { +runIteratorSuite({ lexint: false }) +runIteratorSuite({ lexint: true }) + +function runIteratorSuite (opts) { + var tag = '(' + (opts.lexint ? 'lexint' : 'hash') + ') ' + tape(tag + 'basic iteration', function (t) { + var db = create.one(null, opts) + var vals = ['a', 'b', 'c'] + var expected = toMap(vals) + + put(db, vals, function (err) { t.error(err, 'no error') - t.same(map, expected, 'iterated all values') - t.end() + all(db.iterator(), function (err, map) { + t.error(err, 'no error') + t.same(map, expected, 'iterated all values') + t.end() + }) }) }) -}) -tape('iterate a big db', function (t) { - var db = create.one() + tape(tag + 'iterate a big db', function (t) { + var db = create.one(null, opts) - var vals = range(1000, '#') - var expected = toMap(vals) + var vals = range(1000, '#') + var expected = toMap(vals) - put(db, vals, function (err) { - t.error(err, 'no error') - all(db.iterator(), function (err, map) { + put(db, vals, function (err) { t.error(err, 'no error') - t.same(map, expected, 'iterated all values') - t.end() + all(db.iterator(), function (err, map) { + t.error(err, 'no error') + t.same(map, expected, 'iterated all values') + t.end() + }) }) }) -}) -tape('prefix basic iteration', function (t) { - var db = create.one() - var vals = ['foo/a', 'foo/b', 'foo/c'] - var expected = toMap(vals) + tape(tag + 'prefix basic iteration', function (t) { + var db = create.one(null, opts) + var vals = ['foo/a', 'foo/b', 'foo/c'] + var expected = toMap(vals) - vals = vals.concat(['a', 'b', 'c']) + vals = vals.concat(['a', 'b', 'c']) - put(db, vals, function (err) { - t.error(err, 'no error') - all(db.iterator('foo'), function (err, map) { + put(db, vals, function (err) { t.error(err, 'no error') - t.same(map, expected, 'iterated all values') - t.end() + all(db.iterator('foo'), function (err, map) { + t.error(err, 'no error') + t.same(map, expected, 'iterated all values') + t.end() + }) }) }) -}) -tape('empty prefix iteration', function (t) { - var db = create.one() - var vals = ['foo/a', 'foo/b', 'foo/c'] - var expected = {} + tape(tag + 'empty prefix iteration', function (t) { + var db = create.one(null, opts) + var vals = ['foo/a', 'foo/b', 'foo/c'] + var expected = {} - put(db, vals, function (err) { - t.error(err, 'no error') - all(db.iterator('bar'), function (err, map) { + put(db, vals, function (err) { t.error(err, 'no error') - t.same(map, expected, 'iterated all values') - t.end() + all(db.iterator('bar'), function (err, map) { + t.error(err, 'no error') + t.same(map, expected, 'iterated all values') + t.end() + }) }) }) -}) -tape('prefix iterate a big db', function (t) { - var db = create.one() + tape(tag + 'prefix iterate a big db', function (t) { + var db = create.one(null, opts) - var vals = range(1000, 'foo/#') - var expected = toMap(vals) + var vals = range(1000, 'foo/#') + var expected = toMap(vals) - vals = vals.concat(range(1000, '#')) + vals = vals.concat(range(1000, '#')) - put(db, vals, function (err) { - t.error(err, 'no error') - all(db.iterator('foo'), function (err, map) { + put(db, vals, function (err) { t.error(err, 'no error') - t.same(map, expected, 'iterated all values') - t.end() + all(db.iterator('foo'), function (err, map) { + t.error(err, 'no error') + t.same(map, expected, 'iterated all values') + t.end() + }) }) }) -}) - -tape('non recursive iteration', function (t) { - var db = create.one() - - var vals = [ - 'a', - 'a/b/c/d', - 'a/c', - 'b', - 'b/b/c', - 'c/a', - 'c' - ] - - put(db, vals, function (err) { - t.error(err, 'no error') - all(db.iterator({recursive: false}), function (err, map) { + + tape(tag + 'non recursive iteration', function (t) { + var db = create.one(null, opts) + + var vals = [ + 'a', + 'a/b/c/d', + 'a/c', + 'b', + 'b/b/c', + 'c/a', + 'c' + ] + + put(db, vals, function (err) { t.error(err, 'no error') - var keys = Object.keys(map).map(k => k.split('/')[0]) - t.same(keys.sort(), ['a', 'b', 'c'], 'iterated all values') - t.end() + all(db.iterator({recursive: false}), function (err, map) { + t.error(err, 'no error') + var keys = Object.keys(map).map(k => k.split('/')[0]) + t.same(keys.sort(), ['a', 'b', 'c'], 'iterated all values') + t.end() + }) }) }) -}) -tape('mixed nested and non nexted iteration', function (t) { - var db = create.one() - var vals = ['a', 'a/a', 'a/b', 'a/c', 'a/a/a', 'a/a/b', 'a/a/c'] - var expected = toMap(vals) + tape(tag + 'mixed nested and non nexted iteration', function (t) { + var db = create.one(null, opts) + var vals = ['a', 'a/a', 'a/b', 'a/c', 'a/a/a', 'a/a/b', 'a/a/c'] + var expected = toMap(vals) - put(db, vals, function (err) { - t.error(err, 'no error') - all(db.iterator(), function (err, map) { + put(db, vals, function (err) { t.error(err, 'no error') - t.same(map, expected, 'iterated all values') - t.end() + all(db.iterator(), function (err, map) { + t.error(err, 'no error') + t.same(map, expected, 'iterated all values') + t.end() + }) }) }) -}) - -tape('two writers, simple fork', function (t) { - t.plan(2 * 2 + 1) - - create.two(function (db1, db2, replicate) { - run( - cb => db1.put('0', '0', cb), - replicate, - cb => db1.put('1', '1a', cb), - cb => db2.put('1', '1b', cb), - cb => db1.put('10', '10', cb), - replicate, - cb => db1.put('2', '2', cb), - cb => db1.put('1/0', '1/0', cb), - done - ) - - function done (err) { - t.error(err, 'no error') - all(db1.iterator(), ondb1all) - all(db2.iterator(), ondb2all) - } - function ondb2all (err, map) { - t.error(err, 'no error') - t.same(map, {'0': ['0'], '1': ['1a', '1b'], '10': ['10']}) - } + tape(tag + 'two writers, simple fork', function (t) { + t.plan(2 * 2 + 1) + + create.two(opts, function (db1, db2, replicate) { + run( + cb => db1.put('0', '0', cb), + replicate, + cb => db1.put('1', '1a', cb), + cb => db2.put('1', '1b', cb), + cb => db1.put('10', '10', cb), + replicate, + cb => db1.put('2', '2', cb), + cb => db1.put('1/0', '1/0', cb), + done + ) + + function done (err) { + t.error(err, 'no error') + all(db1.iterator(), ondb1all) + all(db2.iterator(), ondb2all) + } - function ondb1all (err, map) { - t.error(err, 'no error') - t.same(map, {'0': ['0'], '1': ['1a', '1b'], '10': ['10'], '2': ['2'], '1/0': ['1/0']}) - } - }) -}) - -tape('two writers, one fork', function (t) { - create.two(function (db1, db2, replicate) { - run( - cb => db1.put('0', '0', cb), - cb => db2.put('2', '2', cb), - cb => db2.put('3', '3', cb), - cb => db2.put('4', '4', cb), - cb => db2.put('5', '5', cb), - cb => db2.put('6', '6', cb), - cb => db2.put('7', '7', cb), - cb => db2.put('8', '8', cb), - cb => db2.put('9', '9', cb), - cb => replicate(cb), - cb => db1.put('1', '1a', cb), - cb => db2.put('1', '1b', cb), - cb => replicate(cb), - cb => db1.put('0', '00', cb), - cb => replicate(cb), - cb => db2.put('hi', 'ho', cb), - done - ) - - function done (err) { - t.error(err, 'no error') - all(db1.iterator(), function (err, vals) { + function ondb2all (err, map) { t.error(err, 'no error') - t.same(vals, { - '0': ['00'], - '1': ['1a', '1b'], - '2': ['2'], - '3': ['3'], - '4': ['4'], - '5': ['5'], - '6': ['6'], - '7': ['7'], - '8': ['8'], - '9': ['9'] - }) + t.same(map, {'0': ['0'], '1': ['1a', '1b'], '10': ['10']}) + } - all(db2.iterator(), function (err, vals) { + function ondb1all (err, map) { + t.error(err, 'no error') + t.same(map, {'0': ['0'], '1': ['1a', '1b'], '10': ['10'], '2': ['2'], '1/0': ['1/0']}) + } + }) + }) + + tape(tag + 'two writers, one fork', function (t) { + create.two(opts, function (db1, db2, replicate) { + run( + cb => db1.put('0', '0', cb), + cb => db2.put('2', '2', cb), + cb => db2.put('3', '3', cb), + cb => db2.put('4', '4', cb), + cb => db2.put('5', '5', cb), + cb => db2.put('6', '6', cb), + cb => db2.put('7', '7', cb), + cb => db2.put('8', '8', cb), + cb => db2.put('9', '9', cb), + cb => replicate(cb), + cb => db1.put('1', '1a', cb), + cb => db2.put('1', '1b', cb), + cb => replicate(cb), + cb => db1.put('0', '00', cb), + cb => replicate(cb), + cb => db2.put('hi', 'ho', cb), + done + ) + + function done (err) { + t.error(err, 'no error') + all(db1.iterator(), function (err, vals) { t.error(err, 'no error') t.same(vals, { '0': ['00'], @@ -208,148 +198,164 @@ tape('two writers, one fork', function (t) { '6': ['6'], '7': ['7'], '8': ['8'], - '9': ['9'], - 'hi': ['ho'] + '9': ['9'] + }) + + all(db2.iterator(), function (err, vals) { + t.error(err, 'no error') + t.same(vals, { + '0': ['00'], + '1': ['1a', '1b'], + '2': ['2'], + '3': ['3'], + '4': ['4'], + '5': ['5'], + '6': ['6'], + '7': ['7'], + '8': ['8'], + '9': ['9'], + 'hi': ['ho'] + }) + t.end() }) - t.end() }) - }) - } + } + }) }) -}) - -tape('two writers, one fork, many values', function (t) { - var r = range(100, 'i') - - create.two(function (db1, db2, replicate) { - run( - cb => db1.put('0', '0', cb), - cb => db2.put('2', '2', cb), - cb => db2.put('3', '3', cb), - cb => db2.put('4', '4', cb), - cb => db2.put('5', '5', cb), - cb => db2.put('6', '6', cb), - cb => db2.put('7', '7', cb), - cb => db2.put('8', '8', cb), - cb => db2.put('9', '9', cb), - cb => replicate(cb), - cb => db1.put('1', '1a', cb), - cb => db2.put('1', '1b', cb), - cb => replicate(cb), - cb => db1.put('0', '00', cb), - r.map(i => cb => db1.put(i, i, cb)), - cb => replicate(cb), - done - ) - - function done (err) { - t.error(err, 'no error') - var expected = { - '0': ['00'], - '1': ['1a', '1b'], - '2': ['2'], - '3': ['3'], - '4': ['4'], - '5': ['5'], - '6': ['6'], - '7': ['7'], - '8': ['8'], - '9': ['9'] - } + tape(tag + 'two writers, one fork, many values', function (t) { + var r = range(100, 'i') + + create.two(opts, function (db1, db2, replicate) { + run( + cb => db1.put('0', '0', cb), + cb => db2.put('2', '2', cb), + cb => db2.put('3', '3', cb), + cb => db2.put('4', '4', cb), + cb => db2.put('5', '5', cb), + cb => db2.put('6', '6', cb), + cb => db2.put('7', '7', cb), + cb => db2.put('8', '8', cb), + cb => db2.put('9', '9', cb), + cb => replicate(cb), + cb => db1.put('1', '1a', cb), + cb => db2.put('1', '1b', cb), + cb => replicate(cb), + cb => db1.put('0', '00', cb), + r.map(i => cb => db1.put(i, i, cb)), + cb => replicate(cb), + done + ) + + function done (err) { + t.error(err, 'no error') - r.forEach(function (v) { - expected[v] = [v] - }) + var expected = { + '0': ['00'], + '1': ['1a', '1b'], + '2': ['2'], + '3': ['3'], + '4': ['4'], + '5': ['5'], + '6': ['6'], + '7': ['7'], + '8': ['8'], + '9': ['9'] + } - all(db1.iterator(), function (err, vals) { - t.error(err, 'no error') - t.same(vals, expected) - all(db2.iterator(), function (err, vals) { + r.forEach(function (v) { + expected[v] = [v] + }) + + all(db1.iterator(), function (err, vals) { t.error(err, 'no error') t.same(vals, expected) - t.end() + all(db2.iterator(), function (err, vals) { + t.error(err, 'no error') + t.same(vals, expected) + t.end() + }) }) - }) - } + } + }) }) -}) - -tape('two writers, fork', function (t) { - t.plan(2 * 2 + 1) - - create.two(function (a, b, replicate) { - run( - cb => a.put('a', 'a', cb), - replicate, - cb => b.put('a', 'b', cb), - cb => a.put('b', 'c', cb), - replicate, - done - ) - - function done (err) { - t.error(err, 'no error') - all(a.iterator(), onall) - all(b.iterator(), onall) + tape(tag + 'two writers, fork', function (t) { + t.plan(2 * 2 + 1) + + create.two(opts, function (a, b, replicate) { + run( + cb => a.put('a', 'a', cb), + replicate, + cb => b.put('a', 'b', cb), + cb => a.put('b', 'c', cb), + replicate, + done + ) - function onall (err, map) { + function done (err) { t.error(err, 'no error') - t.same(map, {b: ['c'], a: ['b']}) + + all(a.iterator(), onall) + all(b.iterator(), onall) + + function onall (err, map) { + t.error(err, 'no error') + t.same(map, {b: ['c'], a: ['b']}) + } } - } + }) }) -}) - -tape('three writers, two forks', function (t) { - t.plan(2 * 3 + 1) - - var replicate = require('./helpers/replicate') - - create.three(function (a, b, c, replicateAll) { - run( - cb => a.put('a', 'a', cb), - replicateAll, - cb => b.put('a', 'ab', cb), - cb => a.put('some', 'some', cb), - cb => replicate(a, c, cb), - cb => c.put('c', 'c', cb), - replicateAll, - done - ) - - function done (err) { - t.error(err, 'no error') - all(a.iterator(), onall) - all(b.iterator(), onall) - all(c.iterator(), onall) - function onall (err, map) { + tape(tag + 'three writers, two forks', function (t) { + t.plan(2 * 3 + 1) + + var replicate = require('./helpers/replicate') + + create.three(opts, function (a, b, c, replicateAll) { + run( + cb => a.put('a', 'a', cb), + replicateAll, + cb => b.put('a', 'ab', cb), + cb => a.put('some', 'some', cb), + cb => replicate(a, c, cb), + cb => c.put('c', 'c', cb), + replicateAll, + done + ) + + function done (err) { t.error(err, 'no error') - t.same(map, {a: ['ab'], c: ['c'], some: ['some']}) + all(a.iterator(), onall) + all(b.iterator(), onall) + all(c.iterator(), onall) + + function onall (err, map) { + t.error(err, 'no error') + t.same(map, {a: ['ab'], c: ['c'], some: ['some']}) + } } - } + }) }) -}) -tape('list buffers an iterator', function (t) { - var db = create.one() + tape(tag + 'list buffers an iterator', function (t) { + var db = create.one(null, opts) - put(db, ['a', 'b', 'b/c'], function (err) { - t.error(err, 'no error') - db.list(function (err, all) { + put(db, ['a', 'b', 'b/c'], function (err) { t.error(err, 'no error') - t.same(all.map(v => v.key).sort(), ['a', 'b', 'b/c']) - db.list('b', {gt: true}, function (err, all) { + db.list(function (err, all) { t.error(err, 'no error') - t.same(all.length, 1) - t.same(all[0].key, 'b/c') - t.end() + t.same(all.map(v => v.key).sort(), ['a', 'b', 'b/c']) + db.list('b', {gt: true}, function (err, all) { + t.error(err, 'no error') + t.same(all.length, 1) + t.same(all[0].key, 'b/c') + t.end() + }) }) }) }) -}) +} function range (n, v) { // #0, #1, #2, ...