getMany performance tests

.gitignore

@@ -36,3 +36,5 @@ test/test-repo-for*
 docs
 
 test/test-repo/datastore
+
+*.flamegraph

README.md

@@ -83,6 +83,35 @@ src
         └── index.js
 ```
 
+## Performance tests
+
+You can run performance tests like this:
+
+```
+$ npm run benchmarks
+```
+
+### Profiling
+
+You can run each of the individual performance tests with a profiler like 0x.
+
+To do that, you need to install 0x:
+
+```bash
+$ npm install 0x --global
+```
+
+And then run the test:
+
+```bash
+$ 0x test/benchmarks/get-many
+```
+
+This will output a flame graph and print the location for it.
+Use the browser Chrome to open and inspect the generated graph.
+
+![Flame graph](https://ipfs.io/ipfs/QmVbyLgYfkLewNtzTAFwAEMmP2hTJgs8sSqsRTBNBjyQ1y)
+
 ## Contribute
 
 Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/js-ipfs-bitswap/issues)!

package.json

@@ -18,7 +18,8 @@
     "bench": "node benchmarks/index",
     "build": "aegir build",
     "coverage": "aegir coverage --provider codecov",
-    "docs": "aegir docs"
+    "docs": "aegir docs",
+    "benchmarks": "node test/benchmarks/get-many"
   },
   "repository": {
     "type": "git",
@@ -57,7 +58,8 @@
     "peer-info": "~0.11.4",
     "pre-commit": "^1.2.2",
     "rimraf": "^2.6.2",
-    "safe-buffer": "^5.1.1"
+    "safe-buffer": "^5.1.1",
+    "stats-lite": "^2.1.0"
   },
   "dependencies": {
     "async": "^2.6.0",

test/benchmarks/get-many.js

@@ -0,0 +1,12 @@
+'use strict'
+
+const distributionTest = require('../utils/distribution-test')
+const print = require('./helpers/print-swarm-results')
+
+print('10 nodes, 10 blocks, 5 iterations', distributionTest(10, 10, 5, (err) => {
+  if (err) {
+    throw err
+  }
+
+  console.log('Finished. Can kill now...')
+}))

test/benchmarks/helpers/print-swarm-results.js

@@ -0,0 +1,36 @@
+'use strict'
+
+const stats = require('stats-lite')
+
+module.exports = (suite, emitter) => {
+  const elapseds = []
+  emitter.once('start', () => {
+    console.log('\n------------------------')
+    console.log(suite)
+    console.log('started')
+  })
+  emitter.once('all connected', () => {
+    console.log('all nodes connected to each other')
+  })
+  emitter.on('getting many', () => {
+    process.stdout.write('.')
+  })
+  emitter.once('stop', () => {
+    console.log('\nstopping')
+  })
+  emitter.once('stopped', () => {
+    console.log('stopped')
+    console.log('stats:')
+    console.log('---------')
+    console.log('mean: %s', stats.mean(elapseds))
+    console.log('median: %s', stats.median(elapseds))
+    console.log('variance: %s', stats.variance(elapseds))
+    console.log('standard deviation: %s', stats.stdev(elapseds))
+    console.log('85th percentile: %s', stats.percentile(elapseds, 0.85))
+  })
+
+  emitter.on('got block', (elapsed) => {
+    process.stdout.write('+')
+    elapseds.push(elapsed)
+  })
+}

test/swarms.js

@@ -2,32 +2,81 @@
 
 /* eslint-env mocha */
 
+const stats = require('stats-lite')
 const distributionTest = require('./utils/distribution-test')
 const test = it
 
 describe('swarms', () => {
+  const print = Boolean(process.env.PRINT)
+
+  after(() => {
+    process.exit()
+  })
+
   test('2 nodes, 2 blocks', function (done) {
     this.timeout(10 * 1000)
-    distributionTest(2, 2, done)
+    maybePrint('2 nodes, 2 blocks', distributionTest(2, 2, done))
   })
 
   test('10 nodes, 2 blocks', function (done) {
     this.timeout(30 * 1000)
-    distributionTest(10, 2, done)
+    maybePrint('10 nodes, 2 blocks', distributionTest(10, 2, done))
+  })
+
+  test.only('10 nodes, 10 blocks', function (done) {
+    this.timeout(30 * 1000)
+    maybePrint('10 nodes, 10 blocks', distributionTest(10, 10, 1, done))
+  })
+
+  test('10 nodes, 20 blocks', function (done) {
+    this.timeout(30 * 1000)
+    maybePrint('10 nodes, 20 blocks', distributionTest(10, 20, done))
   })
 
   test('50 nodes, 2 blocks', function (done) {
     this.timeout(600 * 1000)
-    distributionTest(50, 2, done)
+    maybePrint('50 nodes, 2 blocks', distributionTest(50, 2, done))
   })
 
   test.skip('100 nodes, 2 blocks', function (done) {
     this.timeout(600 * 1000)
-    distributionTest(100, 2, done)
+    maybePrint('100 nodes, 2 blocks', distributionTest(100, 2, done))
   })
 
   test('10 nodes, 100 blocks', function (done) {
     this.timeout(600 * 1000)
-    distributionTest(10, 100, done)
+    maybePrint('10 nodes, 100 blocks', distributionTest(10, 100, done))
   })
+
+  function maybePrint (suite, emitter) {
+    if (!print) {
+      return
+    }
+    const elapseds = []
+    emitter.once('start', () => {
+      console.log('\n------------------------')
+      console.log(suite)
+      console.log('started')
+    })
+    emitter.once('all connected', () => {
+      console.log('all nodes connected to each other')
+    })
+    emitter.once('stop', () => {
+      console.log('stopping')
+    })
+    emitter.once('stopped', () => {
+      console.log('stopped')
+      console.log('stats:')
+      console.log('---------')
+      console.log('mean: %s', stats.mean(elapseds))
+      console.log('median: %s', stats.median(elapseds))
+      console.log('variance: %s', stats.variance(elapseds))
+      console.log('standard deviation: %s', stats.stdev(elapseds))
+      console.log('85th percentile: %s', stats.percentile(elapseds, 0.85))
+    })
+
+    emitter.on('got block', (elapsed) => {
+      elapseds.push(elapsed)
+    })
+  }
 })

test/utils/distribution-test.js

@@ -3,58 +3,86 @@
 const range = require('lodash.range')
 const map = require('async/map')
 const each = require('async/each')
-const parallel = require('async/parallel')
+const whilst = require('async/whilst')
 const series = require('async/series')
 const waterfall = require('async/waterfall')
 const chai = require('chai')
 chai.use(require('dirty-chai'))
 const expect = chai.expect
+const EventEmitter = require('events')
 
 const createBitswap = require('./create-bitswap')
 const makeBlock = require('./make-block')
 const connectAll = require('./connect-all')
 
-module.exports = (instanceCount, blockCount, callback) => {
+module.exports = (instanceCount, blockCount, repeats, callback) => {
+  let pendingRepeats = repeats
   let nodes
-  let blocks
+  const events = new EventEmitter()
 
   waterfall([
-    (cb) => parallel(
-      {
-        nodes: (cb) => map(range(instanceCount), (_, cb) => createBitswap(cb), cb),
-        blocks: (cb) => map(range(blockCount), (_, cb) => makeBlock(cb), cb)
-      },
-      cb),
-    (results, cb) => {
-      nodes = results.nodes
-      blocks = results.blocks
-      const first = nodes[0]
-
-      parallel([
-        (cb) => connectAll(results.nodes, cb),
-        (cb) => each(results.blocks, first.bitswap.put.bind(first.bitswap), cb)
-      ], cb)
+    (cb) => map(range(instanceCount), (_, cb) => createBitswap(cb), cb),
+    (_nodes, cb) => {
+      nodes = _nodes
+      events.emit('start')
+      cb()
     },
-    (results, cb) => {
-      const cids = blocks.map((block) => block.cid)
-      map(nodes, (node, cb) => node.bitswap.getMany(cids, cb), cb)
+    (cb) => {
+      connectAll(nodes, cb)
     },
-    (results, cb) => {
-      try {
-        expect(results).have.lengthOf(instanceCount)
-        results.forEach((nodeBlocks) => {
-          expect(nodeBlocks).to.have.lengthOf(blocks.length)
-          nodeBlocks.forEach((block, i) => {
-            expect(block.data).to.deep.equal(blocks[i].data)
-          })
-        })
-      } catch (err) {
-        return cb(err)
-      }
-      cb()
+    (cb) => {
+      events.emit('all connected')
+      whilst(() => pendingRepeats > 0, (cb) => {
+        const first = nodes[0]
+        let blocks
+        waterfall([
+          (cb) => map(range(blockCount), (_, cb) => makeBlock(cb), cb),
+          (_blocks, cb) => {
+            blocks = _blocks
+            cb()
+          },
+          (cb) => each(blocks, first.bitswap.put.bind(first.bitswap), (err) => {
+            events.emit('first put')
+            cb(err)
+          }),
+          (cb) => map(nodes, (node, cb) => {
+            events.emit('getting many')
+            const cids = blocks.map((block) => block.cid)
+            const start = Date.now()
+            node.bitswap.getMany(cids, (err, result) => {
+              if (err) {
+                cb(err)
+              } else {
+                const elapsed = Date.now() - start
+                events.emit('got block', elapsed)
+                cb(null, result)
+              }
+            })
+          }, cb),
+          (results, cb) => {
+            try {
+              expect(results).have.lengthOf(instanceCount)
+              results.forEach((nodeBlocks) => {
+                expect(nodeBlocks).to.have.lengthOf(blocks.length)
+                nodeBlocks.forEach((block, i) => {
+                  expect(block.data).to.deep.equal(blocks[i].data)
+                })
+              })
+            } catch (err) {
+              return cb(err)
+            }
+            cb()
+          },
+          (cb) => {
+            pendingRepeats--
+            cb()
+          }
+        ], cb)
+      }, cb)
     }
   ],
   (err) => {
+    events.emit('stop')
     each(
       nodes,
       (node, cb) => {
@@ -64,12 +92,14 @@ module.exports = (instanceCount, blockCount, callback) => {
             (cb) => node.libp2pNode.stop(cb),
             (cb) => node.repo.teardown(cb)
           ],
-          cb
-        )
+          cb)
       },
       (err2) => {
+        events.emit('stopped')
         callback(err)
       }
     )
   })
+
+  return events
 }