From 52dbcf2067d6f9dabc61e400a41e3a1ab329c80a Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 22 Apr 2024 14:43:58 +0100 Subject: [PATCH] chore: add transfer benchmark (#90) Adds a benchmark suite for doing various size data transfers between helia and kubo. Closes #88 --- benchmarks/add-dir/src/kubo.ts | 5 +- benchmarks/gc/src/helia.ts | 4 +- benchmarks/gc/src/kubo.ts | 8 +- benchmarks/transfer/package.json | 40 +++++++++ benchmarks/transfer/src/README.md | 42 +++++++++ benchmarks/transfer/src/helia.ts | 77 ++++++++++++++++ benchmarks/transfer/src/index.ts | 143 ++++++++++++++++++++++++++++++ benchmarks/transfer/src/kubo.ts | 90 +++++++++++++++++++ benchmarks/transfer/tsconfig.json | 14 +++ 9 files changed, 411 insertions(+), 12 deletions(-) create mode 100644 benchmarks/transfer/package.json create mode 100644 benchmarks/transfer/src/README.md create mode 100644 benchmarks/transfer/src/helia.ts create mode 100644 benchmarks/transfer/src/index.ts create mode 100644 benchmarks/transfer/src/kubo.ts create mode 100644 benchmarks/transfer/tsconfig.json diff --git a/benchmarks/add-dir/src/kubo.ts b/benchmarks/add-dir/src/kubo.ts index d2e0fc528..7decb5dd4 100644 --- a/benchmarks/add-dir/src/kubo.ts +++ b/benchmarks/add-dir/src/kubo.ts @@ -1,7 +1,7 @@ import { createNode } from 'ipfsd-ctl' import last from 'it-last' import { path as kuboPath } from 'kubo' -import { create as kuboRpcClient } from 'kubo-rpc-client' +import { globSource, create as kuboRpcClient } from 'kubo-rpc-client' import type { CID } from 'multiformats/cid' import fs, { promises as fsPromises } from 'node:fs' import nodePath from 'node:path' @@ -27,8 +27,7 @@ export async function createKuboBenchmark (): Promise { })).cid const addDir = async function (dir: string): Promise { - // @ts-expect-error types are messed up - const res = await last(controller.api.addAll(goRpcClient.globSource(nodePath.dirname(dir), `${nodePath.basename(dir)}/**/*`))) + const res = await last(controller.api.addAll(globSource(nodePath.dirname(dir), `${nodePath.basename(dir)}/**/*`))) if (res == null) { throw new Error('Import failed') diff --git a/benchmarks/gc/src/helia.ts b/benchmarks/gc/src/helia.ts index d5e2ec2cd..abf7170cc 100644 --- a/benchmarks/gc/src/helia.ts +++ b/benchmarks/gc/src/helia.ts @@ -30,7 +30,7 @@ export async function createHeliaBenchmark (): Promise { await drain(helia.blockstore.putMany(map(blocks, ({ key, value }) => ({ cid: key, block: value })))) }, async pin (cid) { - await helia.pins.add(cid) + await drain(helia.pins.add(cid)) }, async teardown () { await helia.stop() @@ -39,7 +39,7 @@ export async function createHeliaBenchmark (): Promise { const pins = await all(helia.pins.ls()) for (const pin of pins) { - await helia.pins.rm(pin.cid) + await drain(helia.pins.rm(pin.cid)) } return pins.length diff --git a/benchmarks/gc/src/kubo.ts b/benchmarks/gc/src/kubo.ts index 5828a848c..e6db8eb63 100644 --- a/benchmarks/gc/src/kubo.ts +++ b/benchmarks/gc/src/kubo.ts @@ -51,13 +51,7 @@ export async function createKuboBenchmark (): Promise { paths: cid })) - const isPinned = result[0].type.includes('direct') || result[0].type.includes('indirect') || result[0].type.includes('recursive') - - if (!isPinned) { - console.info(result) - } - - return isPinned + return result[0].type.includes('direct') || result[0].type.includes('indirect') || result[0].type.includes('recursive') }, hasBlock: async (cid) => { try { diff --git a/benchmarks/transfer/package.json b/benchmarks/transfer/package.json new file mode 100644 index 000000000..45a0ad9f2 --- /dev/null +++ b/benchmarks/transfer/package.json @@ -0,0 +1,40 @@ +{ + "name": "benchmarks-transfer", + "version": "1.0.0", + "main": "index.js", + "private": true, + "type": "module", + "scripts": { + "clean": "aegir clean", + "build": "aegir build --bundle false", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "start": "npm run build && node dist/src/index.js" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^15.0.0", + "@chainsafe/libp2p-yamux": "^6.0.2", + "@helia/unixfs": "^3.0.3", + "@ipld/dag-pb": "^4.0.2", + "@libp2p/websockets": "^8.0.19", + "aegir": "^42.2.5", + "blockstore-fs": "^1.0.1", + "datastore-level": "^10.0.1", + "execa": "^8.0.1", + "helia": "^4.1.0", + "ipfs-unixfs-importer": "^15.1.1", + "ipfsd-ctl": "^14.0.0", + "it-all": "^3.0.1", + "it-buffer-stream": "^3.0.2", + "it-drain": "^3.0.1", + "it-map": "^3.0.2", + "kubo": "^0.28.0", + "kubo-rpc-client": "^4.0.0", + "libp2p": "^1.4.0", + "multiformats": "^13.1.0", + "tinybench": "^2.4.0" + }, + "dependencies": { + "pretty-bytes": "^6.1.0" + } +} diff --git a/benchmarks/transfer/src/README.md b/benchmarks/transfer/src/README.md new file mode 100644 index 000000000..d2201ca8d --- /dev/null +++ b/benchmarks/transfer/src/README.md @@ -0,0 +1,42 @@ +# Transfer Benchmark + +Benchmarks Helia transfer performance against Kubo + +To run: + +1. Add `benchmarks/*` to the `workspaces` entry in the root `package.json` of this repo +2. Run + ```console + $ npm run reset + $ npm i + $ npm run build + $ cd benchmarks/transfer + $ npm start + + > benchmarks-gc@1.0.0 start + > npm run build && node dist/src/index.js + + + > benchmarks-transfer@1.0.0 build + > aegir build --bundle false + + [14:51:28] tsc [started] + [14:51:33] tsc [completed] + generating Ed25519 keypair... + ┌─────────┬────────────────┬─────────┬───────────┬──────┐ + │ (index) │ Implementation │ ops/s │ ms/op │ runs │ + ├─────────┼────────────────┼─────────┼───────────┼──────┤ + //... results here + ``` + +Recently generated graphs: + +- Lower numbers are better +- The legend arrow indicates direction of transfer + - e.g. `helia -> kubo` is the equivalent of + 1. `ipfs.add` executed on Helia + 2. `ipfs.cat` executed on Kubo which pulls the data from Helia + +image + +image diff --git a/benchmarks/transfer/src/helia.ts b/benchmarks/transfer/src/helia.ts new file mode 100644 index 000000000..d5ca95caf --- /dev/null +++ b/benchmarks/transfer/src/helia.ts @@ -0,0 +1,77 @@ +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import { tcp } from '@libp2p/tcp' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import type { TransferBenchmark } from './index.js' +import os from 'node:os' +import path from 'node:path' +import fs from 'node:fs/promises' +import { LevelDatastore } from 'datastore-level' +import { FsBlockstore } from 'blockstore-fs' +import drain from 'it-drain' +import { unixfs } from '@helia/unixfs' +import { identify } from '@libp2p/identify' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import { balanced } from 'ipfs-unixfs-importer/layout' + +export async function createHeliaBenchmark (): Promise { + const repoPath = path.join(os.tmpdir(), `helia-${Math.random()}`) + + const helia = await createHelia({ + blockstore: new FsBlockstore(`${repoPath}/blocks`), + datastore: new LevelDatastore(`${repoPath}/data`), + libp2p: await createLibp2p({ + addresses: { + listen: [ + '/ip4/127.0.0.1/tcp/0' + ] + }, + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + services: { + identify: identify() + }, + connectionManager: { + minConnections: 0 + } + }) + }) + + return { + async teardown () { + await helia.stop() + await fs.rm(repoPath, { + recursive: true, + force: true + }) + }, + async addr () { + return helia.libp2p.getMultiaddrs()[0] + }, + async dial (ma) { + await helia.libp2p.dial(ma) + }, + async add (content, options) { + const fs = unixfs(helia) + + return await fs.addByteStream(content, { + ...options, + chunker: options.chunkSize != null ? fixedSize({ chunkSize: options.chunkSize }) : undefined, + layout: options.maxChildrenPerNode != null ? balanced({ maxChildrenPerNode: options.maxChildrenPerNode }) : undefined + }) + }, + async get (cid) { + const fs = unixfs(helia) + + await drain(fs.cat(cid)) + } + } +} diff --git a/benchmarks/transfer/src/index.ts b/benchmarks/transfer/src/index.ts new file mode 100644 index 000000000..8c8209a37 --- /dev/null +++ b/benchmarks/transfer/src/index.ts @@ -0,0 +1,143 @@ +/* eslint-disable no-console */ + +import type { CID } from 'multiformats/cid' +import { createHeliaBenchmark } from './helia.js' +import { createKuboBenchmark } from './kubo.js' +import bufferStream from 'it-buffer-stream' +import type { Multiaddr } from '@multiformats/multiaddr' +import prettyBytes from 'pretty-bytes' + +const ONE_MEG = 1024 * 1024 + +export interface TransferBenchmark { + teardown: () => Promise + addr: () => Promise + dial: (multiaddr: Multiaddr) => Promise + add: (content: AsyncIterable, options: ImportOptions) => Promise + get: (cid: CID) => Promise +} + +export interface ImportOptions { + cidVersion?: 0 | 1 + rawLeaves?: boolean + chunkSize?: number + maxChildrenPerNode?: number +} + +interface File { + name: string + options: ImportOptions + size: number +} + +const opts: Record = { + 'kubo defaults': { + chunkSize: 256 * 1024, + rawLeaves: false, + cidVersion: 0, + maxChildrenPerNode: 174 + }, + 'filecoin defaults': { + chunkSize: 1024 * 1024, + rawLeaves: true, + cidVersion: 1, + maxChildrenPerNode: 1024 + }, +/* '256KiB block size': { + chunkSize: 256 * 1024, + rawLeaves: true, + cidVersion: 1, + maxChildrenPerNode: 174 + }, + '512KiB block size': { + chunkSize: 256 * 1024 * 2, + rawLeaves: true, + cidVersion: 1, + maxChildrenPerNode: 174 + }, + '1MB block size': { + chunkSize: 1024 * 1024, + rawLeaves: true, + cidVersion: 1, + maxChildrenPerNode: 174 + }, + '2MB block size': { + chunkSize: (1024 * 1024) * 2, + rawLeaves: true, + cidVersion: 1, + maxChildrenPerNode: 174 + }, + '3MB block size': { + chunkSize: (1024 * 1024) * 3, + rawLeaves: true, + cidVersion: 1, + maxChildrenPerNode: 174 + }, + 'Max block size': { + chunkSize: 4193648, + rawLeaves: true, + cidVersion: 1, + maxChildrenPerNode: 174 + } + // Kubo will not sent bitswap messages larger than this +*/ +} + +const tests: Record = {} + +for (const [name, options] of Object.entries(opts)) { + tests[name] = [] + + for (let i = 100; i < 1100; i += 100) { + tests[name].push({ + name: `${i}`, + options, + size: ONE_MEG * i + }) + + console.info(prettyBytes(ONE_MEG * i)) + } +} + +const impls: Array<{ name: string, create: () => Promise }> = [{ + name: 'helia', + create: async () => await createHeliaBenchmark() +}, { + name: 'kubo', + create: async () => await createKuboBenchmark() +}] + +async function main (): Promise { + for (const [name, files] of Object.entries(tests)) { + for (const implA of impls) { + for (const implB of impls) { + console.info(`${implA.name} -> ${implB.name} ${name}`) + + for (const file of files) { + const subjectA = await implA.create() + const subjectB = await implB.create() + + const addr = await subjectB.addr() + await subjectA.dial(addr) + + const cid = await subjectA.add(bufferStream(file.size), file.options) + + const start = Date.now() + + // b pulls from a + await subjectB.get(cid) + + console.info(`${Date.now() - start}`) + + await subjectA.teardown() + await subjectB.teardown() + } + } + } + } +} + +main().catch(err => { + console.error(err) // eslint-disable-line no-console + process.exit(1) +}) diff --git a/benchmarks/transfer/src/kubo.ts b/benchmarks/transfer/src/kubo.ts new file mode 100644 index 000000000..2a7b2df0d --- /dev/null +++ b/benchmarks/transfer/src/kubo.ts @@ -0,0 +1,90 @@ +import drain from 'it-drain' +import type { TransferBenchmark } from './index.js' +import { path as kuboPath } from 'kubo' +import { create as kuboRpcClient, type BlockPutOptions } from 'kubo-rpc-client' +import { createNode } from 'ipfsd-ctl' +import type { ImportOptions } from './index.js' +import { unixfs } from '@helia/unixfs' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import { balanced } from 'ipfs-unixfs-importer/layout' +import * as dagPB from '@ipld/dag-pb' +import * as raw from 'multiformats/codecs/raw' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' + +const FORMAT_LOOKUP: Record = { + [dagPB.code]: 'dag-pb', + [raw.code]: 'raw' +} + +export async function createKuboBenchmark (): Promise { + const controller = await createNode({ + type: 'kubo', + test: true, + bin: kuboPath(), + rpc: kuboRpcClient, + init: { + emptyRepo: true + } + }) + + return { + async teardown () { + await controller.stop() + }, + async addr () { + const id = await controller.api.id() + + return id.addresses[0] + }, + async dial (ma) { + await controller.api.swarm.connect(ma) + }, + async add (content, options: ImportOptions) { + // use Helia's UnixFS tooling to create the DAG otherwise we are limited + // to 1MB block sizes + const fs = unixfs({ + blockstore: { + async get (cid, options = {}) { + return controller.api.block.get(cid, options) + }, + async put (cid, block, options = {}) { + const opts: BlockPutOptions = { + // @ts-expect-error https://github.com/ipfs/js-kubo-rpc-client/pull/227 + allowBigBlock: true + } + + if (cid.version === 1) { + opts.version = 1 + opts.format = FORMAT_LOOKUP[cid.code] + } + + const putCid = await controller.api.block.put(block, opts) + + if (!uint8ArrayEquals(cid.multihash.bytes, putCid.multihash.bytes)) { + throw new Error(`Put failed ${putCid} != ${cid}`) + } + + return cid + }, + async has (cid, options = {}) { + try { + await controller.api.block.get(cid, options) + return true + } catch { + return false + } + } + } + }) + + return await fs.addByteStream(content, { + ...options, + chunker: options.chunkSize != null ? fixedSize({ chunkSize: options.chunkSize }) : undefined, + layout: options.maxChildrenPerNode != null ? balanced({ maxChildrenPerNode: options.maxChildrenPerNode }) : undefined + }) + }, + async get (cid) { + await drain(controller.api.cat(cid)) + } + } +} diff --git a/benchmarks/transfer/tsconfig.json b/benchmarks/transfer/tsconfig.json new file mode 100644 index 000000000..37d51de72 --- /dev/null +++ b/benchmarks/transfer/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../packages/helia" + } + ] +}