diff --git a/.gitignore b/.gitignore index 9547fedb..d23420d9 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ test/test-repo-for* docs test/test-repo/datastore + +*.flamegraph \ No newline at end of file diff --git a/README.md b/README.md index 53dc3449..7506b897 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,35 @@ src └── index.js ``` +## Performance tests + +You can run performance tests like this: + +``` +$ npm run benchmarks +``` + +### Profiling + +You can run each of the individual performance tests with a profiler like 0x. + +To do that, you need to install 0x: + +```bash +$ npm install 0x --global +``` + +And then run the test: + +```bash +$ 0x test/benchmarks/get-many +``` + +This will output a flame graph and print the location for it. +Use the browser Chrome to open and inspect the generated graph. + +![Flame graph](https://ipfs.io/ipfs/QmVbyLgYfkLewNtzTAFwAEMmP2hTJgs8sSqsRTBNBjyQ1y) + ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/js-ipfs-bitswap/issues)! diff --git a/package.json b/package.json index 2f6e73ea..3d1bdcb1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "bench": "node benchmarks/index", "build": "aegir build", "coverage": "aegir coverage --provider codecov", - "docs": "aegir docs" + "docs": "aegir docs", + "benchmarks": "node test/benchmarks/get-many" }, "repository": { "type": "git", @@ -57,7 +58,8 @@ "peer-info": "~0.11.4", "pre-commit": "^1.2.2", "rimraf": "^2.6.2", - "safe-buffer": "^5.1.1" + "safe-buffer": "^5.1.1", + "stats-lite": "^2.1.0" }, "dependencies": { "async": "^2.6.0", diff --git a/test/benchmarks/get-many.js b/test/benchmarks/get-many.js new file mode 100644 index 00000000..40aa1c8d --- /dev/null +++ b/test/benchmarks/get-many.js @@ -0,0 +1,12 @@ +'use strict' + +const distributionTest = require('../utils/distribution-test') +const print = require('./helpers/print-swarm-results') + +print('10 nodes, 10 blocks, 5 iterations', distributionTest(10, 10, 5, (err) => { + if (err) { + throw err + } + + console.log('Finished. Can kill now...') +})) diff --git a/test/benchmarks/helpers/print-swarm-results.js b/test/benchmarks/helpers/print-swarm-results.js new file mode 100644 index 00000000..60ea7b1b --- /dev/null +++ b/test/benchmarks/helpers/print-swarm-results.js @@ -0,0 +1,36 @@ +'use strict' + +const stats = require('stats-lite') + +module.exports = (suite, emitter) => { + const elapseds = [] + emitter.once('start', () => { + console.log('\n------------------------') + console.log(suite) + console.log('started') + }) + emitter.once('all connected', () => { + console.log('all nodes connected to each other') + }) + emitter.on('getting many', () => { + process.stdout.write('.') + }) + emitter.once('stop', () => { + console.log('\nstopping') + }) + emitter.once('stopped', () => { + console.log('stopped') + console.log('stats:') + console.log('---------') + console.log('mean: %s', stats.mean(elapseds)) + console.log('median: %s', stats.median(elapseds)) + console.log('variance: %s', stats.variance(elapseds)) + console.log('standard deviation: %s', stats.stdev(elapseds)) + console.log('85th percentile: %s', stats.percentile(elapseds, 0.85)) + }) + + emitter.on('got block', (elapsed) => { + process.stdout.write('+') + elapseds.push(elapsed) + }) +} diff --git a/test/swarms.js b/test/swarms.js index b88f511f..074b90c1 100644 --- a/test/swarms.js +++ b/test/swarms.js @@ -2,32 +2,81 @@ /* eslint-env mocha */ +const stats = require('stats-lite') const distributionTest = require('./utils/distribution-test') const test = it describe('swarms', () => { + const print = Boolean(process.env.PRINT) + + after(() => { + process.exit() + }) + test('2 nodes, 2 blocks', function (done) { this.timeout(10 * 1000) - distributionTest(2, 2, done) + maybePrint('2 nodes, 2 blocks', distributionTest(2, 2, done)) }) test('10 nodes, 2 blocks', function (done) { this.timeout(30 * 1000) - distributionTest(10, 2, done) + maybePrint('10 nodes, 2 blocks', distributionTest(10, 2, done)) + }) + + test.only('10 nodes, 10 blocks', function (done) { + this.timeout(30 * 1000) + maybePrint('10 nodes, 10 blocks', distributionTest(10, 10, 1, done)) + }) + + test('10 nodes, 20 blocks', function (done) { + this.timeout(30 * 1000) + maybePrint('10 nodes, 20 blocks', distributionTest(10, 20, done)) }) test('50 nodes, 2 blocks', function (done) { this.timeout(600 * 1000) - distributionTest(50, 2, done) + maybePrint('50 nodes, 2 blocks', distributionTest(50, 2, done)) }) test.skip('100 nodes, 2 blocks', function (done) { this.timeout(600 * 1000) - distributionTest(100, 2, done) + maybePrint('100 nodes, 2 blocks', distributionTest(100, 2, done)) }) test('10 nodes, 100 blocks', function (done) { this.timeout(600 * 1000) - distributionTest(10, 100, done) + maybePrint('10 nodes, 100 blocks', distributionTest(10, 100, done)) }) + + function maybePrint (suite, emitter) { + if (!print) { + return + } + const elapseds = [] + emitter.once('start', () => { + console.log('\n------------------------') + console.log(suite) + console.log('started') + }) + emitter.once('all connected', () => { + console.log('all nodes connected to each other') + }) + emitter.once('stop', () => { + console.log('stopping') + }) + emitter.once('stopped', () => { + console.log('stopped') + console.log('stats:') + console.log('---------') + console.log('mean: %s', stats.mean(elapseds)) + console.log('median: %s', stats.median(elapseds)) + console.log('variance: %s', stats.variance(elapseds)) + console.log('standard deviation: %s', stats.stdev(elapseds)) + console.log('85th percentile: %s', stats.percentile(elapseds, 0.85)) + }) + + emitter.on('got block', (elapsed) => { + elapseds.push(elapsed) + }) + } }) diff --git a/test/utils/distribution-test.js b/test/utils/distribution-test.js index dc9efc59..3f27fc36 100644 --- a/test/utils/distribution-test.js +++ b/test/utils/distribution-test.js @@ -3,58 +3,86 @@ const range = require('lodash.range') const map = require('async/map') const each = require('async/each') -const parallel = require('async/parallel') +const whilst = require('async/whilst') const series = require('async/series') const waterfall = require('async/waterfall') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect +const EventEmitter = require('events') const createBitswap = require('./create-bitswap') const makeBlock = require('./make-block') const connectAll = require('./connect-all') -module.exports = (instanceCount, blockCount, callback) => { +module.exports = (instanceCount, blockCount, repeats, callback) => { + let pendingRepeats = repeats let nodes - let blocks + const events = new EventEmitter() waterfall([ - (cb) => parallel( - { - nodes: (cb) => map(range(instanceCount), (_, cb) => createBitswap(cb), cb), - blocks: (cb) => map(range(blockCount), (_, cb) => makeBlock(cb), cb) - }, - cb), - (results, cb) => { - nodes = results.nodes - blocks = results.blocks - const first = nodes[0] - - parallel([ - (cb) => connectAll(results.nodes, cb), - (cb) => each(results.blocks, first.bitswap.put.bind(first.bitswap), cb) - ], cb) + (cb) => map(range(instanceCount), (_, cb) => createBitswap(cb), cb), + (_nodes, cb) => { + nodes = _nodes + events.emit('start') + cb() }, - (results, cb) => { - const cids = blocks.map((block) => block.cid) - map(nodes, (node, cb) => node.bitswap.getMany(cids, cb), cb) + (cb) => { + connectAll(nodes, cb) }, - (results, cb) => { - try { - expect(results).have.lengthOf(instanceCount) - results.forEach((nodeBlocks) => { - expect(nodeBlocks).to.have.lengthOf(blocks.length) - nodeBlocks.forEach((block, i) => { - expect(block.data).to.deep.equal(blocks[i].data) - }) - }) - } catch (err) { - return cb(err) - } - cb() + (cb) => { + events.emit('all connected') + whilst(() => pendingRepeats > 0, (cb) => { + const first = nodes[0] + let blocks + waterfall([ + (cb) => map(range(blockCount), (_, cb) => makeBlock(cb), cb), + (_blocks, cb) => { + blocks = _blocks + cb() + }, + (cb) => each(blocks, first.bitswap.put.bind(first.bitswap), (err) => { + events.emit('first put') + cb(err) + }), + (cb) => map(nodes, (node, cb) => { + events.emit('getting many') + const cids = blocks.map((block) => block.cid) + const start = Date.now() + node.bitswap.getMany(cids, (err, result) => { + if (err) { + cb(err) + } else { + const elapsed = Date.now() - start + events.emit('got block', elapsed) + cb(null, result) + } + }) + }, cb), + (results, cb) => { + try { + expect(results).have.lengthOf(instanceCount) + results.forEach((nodeBlocks) => { + expect(nodeBlocks).to.have.lengthOf(blocks.length) + nodeBlocks.forEach((block, i) => { + expect(block.data).to.deep.equal(blocks[i].data) + }) + }) + } catch (err) { + return cb(err) + } + cb() + }, + (cb) => { + pendingRepeats-- + cb() + } + ], cb) + }, cb) } ], (err) => { + events.emit('stop') each( nodes, (node, cb) => { @@ -64,12 +92,14 @@ module.exports = (instanceCount, blockCount, callback) => { (cb) => node.libp2pNode.stop(cb), (cb) => node.repo.teardown(cb) ], - cb - ) + cb) }, (err2) => { + events.emit('stopped') callback(err) } ) }) + + return events }