Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: benchmark Protobuf-ES #89

Merged
merged 11 commits into from
Oct 23, 2023
27 changes: 13 additions & 14 deletions packages/protons-benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,17 @@

## Install

```console
$ npm i protons-benchmark
```

```console
$ git clone git@github.com:ipfs/protons.git
$ cd protons
$ npm i
$ npm run build
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Without this:

$ npm start                     

> protons-benchmark@0.0.0 prestart
> npm run build


> protons-benchmark@0.0.0 build
> aegir build --no-bundle && cp -R src/protobufjs dist/src/protobufjs

[22:08:47] tsc [started]
src/protons/bench.ts:6:68 - error TS2307: Cannot find module 'protons-runtime' or its corresponding type declarations.

6 import { encodeMessage, decodeMessage, message, enumeration } from 'protons-runtime'

...

Found 89 errors in 2 files.

Errors  Files
    32  src/protons/bench.ts:6
    57  src/protons/rpc.ts:6
[22:08:48] tsc [failed]
[22:08:48] → Command failed with exit code 1: tsc

...

$ cd packages/protons-benchmark
```

## Usage

Run the benchmark suite:
Run the benchmark suite in node:

```console
$ npm start
Expand All @@ -48,10 +45,11 @@ $ npm start
> protons-benchmark@0.0.0 start
> node dist/src/index.js

pbjs x 11,798 ops/sec ±4.58% (88 runs sampled)
protons x 11,693 ops/sec ±2.69% (85 runs sampled)
protobuf.js x 12,419 ops/sec ±1.66% (88 runs sampled)
@protobuf-ts x 10,536 ops/sec ±3.14% (85 runs sampled)
pbjs x 19,188 ops/sec ±0.38% (98 runs sampled)
protons x 19,001 ops/sec ±0.33% (95 runs sampled)
protobuf.js x 19,558 ops/sec ±0.30% (91 runs sampled)
@protobuf-ts x 17,216 ops/sec ±0.32% (95 runs sampled)
protobuf-es x 15,673 ops/sec ±0.48% (93 runs sampled)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ℹ️ Slowest.

Fastest is protobuf.js
```

Expand All @@ -64,11 +62,12 @@ $ npm run start:browser
> npx playwright-test dist/src/index.js --runner benchmark

✔ chromium set up
pbjs x 19,027 ops/sec ±0.86% (67 runs sampled)
protons x 18,901 ops/sec ±0.65% (67 runs sampled)
protobuf.js x 18,937 ops/sec ±0.55% (65 runs sampled)
@protobuf-ts x 16,669 ops/sec ±0.49% (68 runs sampled)
Fastest is pbjs,protobuf.js
pbjs x 19,763 ops/sec ±0.35% (67 runs sampled)
protons x 19,617 ops/sec ±0.37% (68 runs sampled)
protobuf.js x 19,772 ops/sec ±0.34% (67 runs sampled)
@protobuf-ts x 17,204 ops/sec ±0.33% (69 runs sampled)
protobuf-es x 16,032 ops/sec ±0.38% (68 runs sampled)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ℹ️ Slowest.

Fastest is protobuf.js,pbjs
```

## API Docs
Expand Down
6 changes: 6 additions & 0 deletions packages/protons-benchmark/buf.gen.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
version: v1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a way for this config file to live in the protobuf-es folder? It'd be more manageable not to have implementation-specific files outside of their respective directories.

plugins:
- name: es
path: ../../node_modules/.bin/protoc-gen-es
opt: target=ts
out: ./src/protobuf-es
7 changes: 5 additions & 2 deletions packages/protons-benchmark/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,22 @@
"ignorePatterns": [
"src/pbjs/*",
"src/protobuf-ts/*",
"src/protobufjs/*"
"src/protobufjs/*",
"src/protobuf-es/*"
]
},
"scripts": {
"clean": "aegir clean",
"lint": "aegir lint",
"dep-check": "aegir dep-check --ignore @protobuf-ts/plugin pbjs protons",
"dep-check": "aegir dep-check --ignore @protobuf-ts/plugin pbjs protons @bufbuild/protoc-gen-es",
"build": "aegir build --no-bundle && cp -R src/protobufjs dist/src/protobufjs",
"prestart": "npm run build",
"start": "node dist/src/index.js",
"start:browser": "npx playwright-test dist/src/index.js --runner benchmark"
},
"dependencies": {
"@bufbuild/protobuf": "^1.0.0",
"@bufbuild/protoc-gen-es": "^1.3.3",
"@protobuf-ts/plugin": "^2.8.1",
"@protobuf-ts/runtime": "^2.8.1",
"@types/benchmark": "^2.1.1",
Expand Down
10 changes: 10 additions & 0 deletions packages/protons-benchmark/src/buf.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
version: v1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

breaking:
use:
- FILE
lint:
use:
- DEFAULT
except:
- ENUM_ZERO_VALUE_SUFFIX
- ENUM_VALUE_PREFIX
4 changes: 4 additions & 0 deletions packages/protons-benchmark/src/decode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ $ npx playwright-test dist/src/index.js --runner benchmark

import Benchmark from 'benchmark'
import { decodeTest as pbjsDecodeTest } from './pbjs/bench.js'
import { Test as ProtobufEsTest } from './protobuf-es/bench_pb.js'
import { Test as ProtobufjsTest } from './protobufjs/bench.js'
import { Test as ProtonsTest } from './protons/bench.js'

Expand Down Expand Up @@ -36,6 +37,9 @@ new Benchmark.Suite()
.add('protobufjs', () => {
ProtobufjsTest.decode(buf)
})
.add('protobufes', () => {
ProtobufEsTest.fromBinary(buf)
})
.on('error', (err: Error) => {
console.error(err)
})
Expand Down
4 changes: 4 additions & 0 deletions packages/protons-benchmark/src/encode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ $ npx playwright-test dist/src/index.js --runner benchmark

import Benchmark from 'benchmark'
import { encodeTest as pbjsEncodeTest } from './pbjs/bench.js'
import { Test as ProtobufEsTest } from './protobuf-es/bench_pb.js'
import { Test as ProtobufjsTest } from './protobufjs/bench.js'
import { Test as ProtonsTest } from './protons/bench.js'

Expand Down Expand Up @@ -34,6 +35,9 @@ new Benchmark.Suite()
.add('protobufjs', () => {
ProtobufjsTest.encode(message).finish()
})
.add('protobufes', () => {
new ProtobufEsTest(message).toBinary()
})
.on('error', (err: Error) => {
console.error(err)
})
Expand Down
7 changes: 7 additions & 0 deletions packages/protons-benchmark/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ $ npx playwright-test dist/src/index.js --runner benchmark
import { expect } from 'aegir/chai'
import Benchmark from 'benchmark'
import { encodeTest as pbjsEncodeTest, decodeTest as pbjsDecodeTest } from './pbjs/bench.js'
import { Test as ProtobufEsTest } from './protobuf-es/bench_pb.js'
import { Test as ProtobufTsTest } from './protobuf-ts/bench.js'
import { Test as ProtobufjsTest } from './protobufjs/bench.js'
import { Test as ProtonsTest } from './protons/bench.js'
Expand Down Expand Up @@ -59,6 +60,12 @@ new Benchmark.Suite()

expectDecodedCorrectly(result)
})
.add('protobuf-es', () => {
const buf = new ProtobufEsTest(message).toBinary()
const result = ProtobufEsTest.fromBinary(buf)

expectDecodedCorrectly(result)
})
.on('error', (err: Error) => {
console.error(err)
})
Expand Down
Loading