Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run benchmark script once #2778

Merged
merged 8 commits into from
Jun 30, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
@@ -16,6 +16,8 @@ on:
jobs:
run:
runs-on: ubuntu-latest
# Don't run on forks. Forks don't have access to the S3 credentials and the workflow will fail
if: github.event.pull_request.head.repo.full_name == github.repository

steps:
# <common-build> - Uses YAML anchors in the future
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -24,7 +24,8 @@
"test:spec-min": "lerna run test:spec-min --no-bail",
"test:spec-fast": "lerna run test:spec-fast --no-bail",
"test:spec-main": "lerna run test:spec-main --no-bail",
"benchmark": "lerna run benchmark --no-prefix",
"benchmark": "node --max-old-space-size=4096 -r ts-node/register -r ./packages/lodestar/test/setup.ts ./node_modules/.bin/benchmark 'packages/*/test/perf/**/*.test.ts'",
"benchmark:local": "yarn benchmark --local",
"cli": "node --trace-deprecation --max-old-space-size=8192 ./packages/cli/bin/lodestar",
"publish:release": "lerna publish from-package --yes --no-verify-access",
"release": "lerna version --no-push --sign-git-commit",
3 changes: 1 addition & 2 deletions packages/beacon-state-transition/.mocharc.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
colors: true
require:
- ts-node/register
- ./test/mocha-hooks.ts
file: ./test/setup.ts
- ./test/setup.ts
timeout: 5000
exit: true
3 changes: 1 addition & 2 deletions packages/beacon-state-transition/package.json
Original file line number Diff line number Diff line change
@@ -32,8 +32,7 @@
"check-types": "tsc",
"lint": "eslint --color --ext .ts src/ test/",
"lint:fix": "yarn run lint --fix",
"test:unit": "mocha 'test/unit/**/*.test.ts'",
"benchmark": "node --max-old-space-size=4096 -r ts-node/register ../../node_modules/.bin/benchmark 'test/perf/**/*.test.ts'"
"test:unit": "mocha 'test/unit/**/*.test.ts'"
},
"types": "lib/index.d.ts",
"dependencies": {
10 changes: 0 additions & 10 deletions packages/beacon-state-transition/test/mocha-hooks.ts

This file was deleted.

Original file line number Diff line number Diff line change
@@ -24,11 +24,14 @@ describe("root equals", () => {
ssz.Root.equals(rootTree, stateRoot);
});

itBench("ssz.Root.equals with valueOf()", () => {
ssz.Root.equals(rootTree.valueOf() as Uint8Array, stateRoot);
});
// We only ssz.Root.equals(), benchmark only that one in CI
if (!process.env.CI) {
itBench("ssz.Root.equals with valueOf()", () => {
ssz.Root.equals(rootTree.valueOf() as Uint8Array, stateRoot);
});

itBench("byteArrayEquals with valueOf()", () => {
byteArrayEquals(rootTree.valueOf() as Uint8Array, stateRoot);
});
itBench("byteArrayEquals with valueOf()", () => {
byteArrayEquals(rootTree.valueOf() as Uint8Array, stateRoot);
});
}
});
1 change: 1 addition & 0 deletions packages/beacon-state-transition/test/setup.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {init} from "@chainsafe/bls";

// blst-native initialization is syncronous
// Initialize bls here instead of in before() so it's available inside describe() blocks
init("blst-native").catch((e) => {
// eslint-disable-next-line no-console
console.error(e);
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import crypto from "crypto";
import bls, {init} from "@chainsafe/bls";
import bls from "@chainsafe/bls";
import {BitList, List, TreeBacked} from "@chainsafe/ssz";
import {config} from "@chainsafe/lodestar-config/default";
import {ValidatorIndex, BLSSignature, ssz} from "@chainsafe/lodestar-types";
@@ -11,10 +11,6 @@ import {phase0, createCachedBeaconState, allForks} from "../../../src";
import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params";

describe("signatureSets", () => {
before("Init BLS", async () => {
await init("blst-native");
});

it("should aggregate all signatures from a block", () => {
const EMPTY_SIGNATURE = Buffer.alloc(96);

5 changes: 3 additions & 2 deletions packages/light-client/.mocharc.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
colors: true
require: ts-node/register
file: ./test/setup.ts
require:
- ts-node/register
- ./test/setup.ts
timeout: 5000
exit: true
3 changes: 1 addition & 2 deletions packages/light-client/package.json
Original file line number Diff line number Diff line change
@@ -33,8 +33,7 @@
"lint:fix": "yarn run lint --fix",
"pretest": "yarn run check-types",
"test": "yarn test:unit && yarn test:e2e",
"test:unit": "LODESTAR_PRESET=minimal nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'",
"test:e2e": "mocha 'test/e2e/**/*.test.ts'"
"test:unit": "LODESTAR_PRESET=minimal nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'"
},
"dependencies": {
"@chainsafe/abort-controller": "^3.0.1",
8 changes: 6 additions & 2 deletions packages/light-client/test/setup.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import {init} from "@chainsafe/bls";

before(async function () {
await init("blst-native");
// blst-native initialization is syncronous
// Initialize bls here instead of in before() so it's available inside describe() blocks
init("blst-native").catch((e) => {
// eslint-disable-next-line no-console
console.error(e);
process.exit(1);
});
5 changes: 3 additions & 2 deletions packages/lodestar/.mocharc.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
colors: true
require: ts-node/register
file: ./test/setup.ts
require:
- ts-node/register
- ./test/setup.ts
timeout: 5000
exit: true
13 changes: 6 additions & 7 deletions packages/lodestar/package.json
Original file line number Diff line number Diff line change
@@ -35,12 +35,12 @@
"lint:fix": "yarn run lint --fix",
"pretest": "yarn run check-types",
"test": "yarn test:unit && yarn test:e2e",
"test:unit": "LODESTAR_PRESET=minimal nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'",
"test:e2e": "LODESTAR_PRESET=minimal mocha 'test/e2e/**/*.test.ts'",
"test:sim": "LODESTAR_PRESET=minimal mocha 'test/sim/**/*.test.ts'",
"test:sim:singleThread": "LODESTAR_PRESET=minimal mocha 'test/sim/singleNodeSingleThread.test.ts'",
"test:sim:singleThreadMultiNode": "LODESTAR_PRESET=minimal mocha 'test/sim/multiNodeSingleThread.test.ts'",
"test:sim:multiThread": "LODESTAR_PRESET=minimal mocha 'test/sim/multiNodeMultiThread.test.ts'"
"test:unit": "nyc --cache-dir .nyc_output/.cache -e .ts mocha 'test/unit/**/*.test.ts'",
"test:e2e": "mocha 'test/e2e/**/*.test.ts'",
"test:sim": "mocha 'test/sim/**/*.test.ts'",
"test:sim:singleThread": "mocha 'test/sim/singleNodeSingleThread.test.ts'",
"test:sim:singleThreadMultiNode": "mocha 'test/sim/multiNodeSingleThread.test.ts'",
"test:sim:multiThread": "mocha 'test/sim/multiNodeMultiThread.test.ts'"
},
"dependencies": {
"@chainsafe/abort-controller": "^3.0.1",
@@ -103,7 +103,6 @@
"@types/supertest": "^2.0.8",
"@types/tmp": "^0.2.0",
"@types/varint": "^5.0.0",
"benchmark": "^2.1.4",
"eventsource": "^1.1.0",
"rewiremock": "^3.14.3",
"rimraf": "^3.0.2",
6 changes: 1 addition & 5 deletions packages/lodestar/test/e2e/chain/bls/multithread.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {expect} from "chai";
import {AbortController} from "@chainsafe/abort-controller";
import {bls, init} from "@chainsafe/bls";
import {bls} from "@chainsafe/bls";
import {ISignatureSet, SignatureSetType} from "@chainsafe/lodestar-beacon-state-transition";
import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread";
import {testLogger} from "../../../utils/logger";
@@ -9,10 +9,6 @@ describe("chain / bls / multithread queue", function () {
this.timeout(30 * 1000);
const logger = testLogger();

before("Init BLS", async () => {
await init("blst-native");
});

let controller: AbortController;
beforeEach(() => (controller = new AbortController()));
afterEach(() => controller.abort());
69 changes: 69 additions & 0 deletions packages/lodestar/test/perf/bls/bls.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import {itBench, setBenchOpts} from "@dapplion/benchmark";
import {bls, PublicKey, SecretKey, Signature} from "@chainsafe/bls";
import {linspace} from "../../../src/util/numpy";

describe("BLS ops", function () {
setBenchOpts({
maxMs: 60 * 1000,
minMs: 5 * 1000,
runs: 1024,
});

type Keypair = {publicKey: PublicKey; secretKey: SecretKey};
type BlsSet = {publicKey: PublicKey; message: Uint8Array; signature: Signature};

// Create and cache (on demand) crypto data to bencharmk
const sets = new Map<number, BlsSet>();
const keypairs = new Map<number, Keypair>();

function getKeypair(i: number): Keypair {
let keypair = keypairs.get(i);
if (!keypair) {
const secretKey = bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1));
const publicKey = secretKey.toPublicKey();
keypair = {secretKey, publicKey};
keypairs.set(i, keypair);
}
return keypair;
}

function getSet(i: number): BlsSet {
let set = sets.get(i);
if (!set) {
const {secretKey, publicKey} = getKeypair(i);
const message = Buffer.alloc(32, i + 1);
set = {publicKey, message: message, signature: secretKey.sign(message)};
sets.set(i, set);
}
return set;
}

itBench({id: `BLS verify - ${bls.implementation}`, beforeEach: () => getSet(0)}, (set) => {
const isValid = set.signature.verify(set.publicKey, set.message);
if (!isValid) throw Error("Invalid");
});

// An aggregate and proof object has 3 signatures.
// We may want to bundle up to 32 sets in a single batch.
for (const count of [3, 8, 32]) {
itBench({
id: `BLS verifyMultipleSignatures ${count} - ${bls.implementation}`,
beforeEach: () => linspace(0, count - 1).map((i) => getSet(i)),
fn: (sets) => {
const isValid = bls.Signature.verifyMultipleSignatures(sets);
if (!isValid) throw Error("Invalid");
},
});
}

// Attestations in Mainnet contain 128 max on average
for (const count of [32, 128]) {
itBench({
id: `BLS aggregatePubkeys ${count} - ${bls.implementation}`,
beforeEach: () => linspace(0, count - 1).map((i) => getKeypair(i).publicKey),
fn: (pubkeys) => {
bls.PublicKey.aggregate(pubkeys);
},
});
}
});
Original file line number Diff line number Diff line change
@@ -1,19 +1,27 @@
import {itBench, setBenchOpts} from "@dapplion/benchmark";
import {config} from "@chainsafe/lodestar-config/default";
import {ssz} from "@chainsafe/lodestar-types";
import {CachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition";
import {allForks, ssz} from "@chainsafe/lodestar-types";
import {generateCachedState} from "../../../utils/state";
import {CheckpointStateCache} from "../../../../src/chain/stateCache";
import {Checkpoint} from "../../../../../types/phase0";

describe("CheckpointStateCache perf tests", function () {
let state: CachedBeaconState<allForks.BeaconState>;
let checkpoint: Checkpoint;
let checkpointStateCache: CheckpointStateCache;

setBenchOpts({
maxMs: 60 * 1000,
minMs: 1 * 1000,
runs: 1024,
});

const checkpointStateCache = new CheckpointStateCache(config);
const state = generateCachedState();
const checkpoint = ssz.phase0.Checkpoint.defaultValue();
before(() => {
checkpointStateCache = new CheckpointStateCache(config);
state = generateCachedState();
checkpoint = ssz.phase0.Checkpoint.defaultValue();
});

itBench("CheckpointStateCache - add get delete", () => {
checkpointStateCache.add(checkpoint, state);
14 changes: 11 additions & 3 deletions packages/lodestar/test/setup.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
import {init} from "@chainsafe/bls";
import {before} from "mocha";

before(async function () {
await init("blst-native");
// Set minimal
if (process.env.LODESTAR_PRESET === undefined) {
process.env.LODESTAR_PRESET = "minimal";
}

// blst-native initialization is syncronous
// Initialize bls here instead of in before() so it's available inside describe() blocks
init("blst-native").catch((e) => {
// eslint-disable-next-line no-console
console.error(e);
process.exit(1);
});
3 changes: 1 addition & 2 deletions packages/lodestar/test/unit/db/syncCommittee.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {bls, init} from "@chainsafe/bls";
import {bls} from "@chainsafe/bls";
import {config} from "@chainsafe/lodestar-config/default";
import {altair} from "@chainsafe/lodestar-types";
import {toHexString} from "@chainsafe/ssz";
@@ -17,7 +17,6 @@ describe("SyncCommitteeCache", function () {
let syncCommittee: altair.SyncCommitteeMessage;

before("Init BLS", async () => {
await init("blst-native");
const sk = bls.SecretKey.fromBytes(Buffer.alloc(32, 1));
syncCommittee = generateSyncCommitteeSignature({
slot,
5 changes: 3 additions & 2 deletions packages/validator/.mocharc.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
colors: true
require: ts-node/register
file: ./test/setup.ts
require:
- ts-node/register
- ./test/setup.ts
8 changes: 6 additions & 2 deletions packages/validator/test/setup.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import {init} from "@chainsafe/bls";

before(async function () {
await init("blst-native");
// blst-native initialization is syncronous
// Initialize bls here instead of in before() so it's available inside describe() blocks
init("blst-native").catch((e) => {
// eslint-disable-next-line no-console
console.error(e);
process.exit(1);
});
15 changes: 1 addition & 14 deletions yarn.lock
Original file line number Diff line number Diff line change
@@ -4299,14 +4299,6 @@ before-after-hook@^2.2.0:
resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.1.tgz#73540563558687586b52ed217dad6a802ab1549c"
integrity sha512-/6FKxSTWoJdbsLDF8tdIjaRiFXiE6UHsEHE3OPI/cwPURCVi1ukP0gmLn7XWEiFk5TcwQjjY5PWsU+j+tgXgmw==

benchmark@^2.1.4:
version "2.1.4"
resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik=
dependencies:
lodash "^4.17.4"
platform "^1.3.3"

bigint-buffer@^1.1.5:
version "1.1.5"
resolved "https://registry.yarnpkg.com/bigint-buffer/-/bigint-buffer-1.1.5.tgz#d038f31c8e4534c1f8d0015209bf34b4fa6dd442"
@@ -9040,7 +9032,7 @@ lodash.throttle@^4.1.1:
resolved "https://registry.yarnpkg.com/lodash.throttle/-/lodash.throttle-4.1.1.tgz#c23e91b710242ac70c37f1e1cda9274cc39bf2f4"
integrity sha1-wj6RtxAkKscMN/HhzaknTMOb8vQ=

lodash@^4.1.2, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.4, lodash@^4.7.0:
lodash@^4.1.2, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.7.0:
version "4.17.21"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -10885,11 +10877,6 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0:
dependencies:
find-up "^4.0.0"

platform@^1.3.3:
version "1.3.5"
resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444"
integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q==

posix-character-classes@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"