Skip to content

Commit

Permalink
Run performance tests in mainnet preset (#2935)
Browse files Browse the repository at this point in the history
* Run performance tests in mainnet preset

* Enforce mainnet preset in perf tests

* Force LODESTAR_PRESET=mainnet in perf tests

* Remove logger statement from perf test

* Use minimum values in attestation sample data

* Increase timeout in getAttestationsForBlock
dapplion authored Aug 10, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent c516647 commit 485ec69
Showing 7 changed files with 70 additions and 52 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@
"test:spec-min": "lerna run test:spec-min --no-bail",
"test:spec-fast": "lerna run test:spec-fast --no-bail",
"test:spec-main": "lerna run test:spec-main --no-bail",
"benchmark": "node --max-old-space-size=4096 -r ts-node/register -r ./packages/lodestar/test/setup.ts ./node_modules/.bin/benchmark 'packages/*/test/perf/**/*.test.ts'",
"benchmark": "LODESTAR_PRESET=mainnet node --max-old-space-size=4096 -r ts-node/register -r ./packages/lodestar/test/setupBLS.ts ./node_modules/.bin/benchmark 'packages/*/test/perf/**/*.test.ts'",
"benchmark:local": "yarn benchmark --local",
"publish:release": "lerna publish from-package --yes --no-verify-access",
"release": "lerna version --no-push --sign-git-commit",
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import {expect} from "chai";
import {ACTIVE_PRESET, PresetName} from "@chainsafe/lodestar-params";
import {beforeProcessEpoch} from "../../src/allForks";
import {generatePerfTestCachedStateAltair, generatePerfTestCachedStatePhase0, perfStateId} from "./util";

describe("Perf test sanity check", function () {
this.timeout(60 * 1000);

if (ACTIVE_PRESET !== PresetName.mainnet) {
throw Error(`ACTIVE_PRESET '${ACTIVE_PRESET}' must be mainnet`);
}

const numValidators = 250000;
const targetStakeYWei = 7;
const targetStake = BigInt(targetStakeYWei) * BigInt(1) ** BigInt(15);
3 changes: 2 additions & 1 deletion packages/lodestar/.mocharc.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
colors: true
require:
- ts-node/register
- ./test/setup.ts
- ./test/setupBLS.ts
- ./test/setupPreset.ts
timeout: 5000
exit: true
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {itBench, setBenchOpts} from "@dapplion/benchmark";
import {expect} from "chai";
import {
allForks,
computeEpochAtSlot,
@@ -8,7 +9,6 @@ import {
import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool";
import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params";
import {List} from "@chainsafe/ssz";
import {profilerLogger} from "@chainsafe/lodestar-beacon-state-transition/test/utils/logger";
import {generatePerfTestCachedStateAltair} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util";

// Jul-21 09:20:46.653 [] info: Number of participations in state previousEpoch=250000, currentEpoch=250000
@@ -22,61 +22,69 @@ describe("getAttestationsForBlock", () => {
runs: 64,
});

const originalState = (generatePerfTestCachedStateAltair({
goBackOneSlot: true,
}) as unknown) as allForks.CachedBeaconState<allForks.BeaconState>;
const stateSlot = originalState.slot;
const logger = profilerLogger();
const numPreviousEpochParticipation = originalState.previousEpochParticipation.persistent
.toArray()
.filter((part) => part && part.timelySource).length;
const numCurrentEpochParticipation = originalState.currentEpochParticipation.persistent
.toArray()
.filter((part) => part && part.timelySource).length;
logger.info("Number of participations in state", {
previousEpoch: numPreviousEpochParticipation,
currentEpoch: numCurrentEpochParticipation,
let originalState: allForks.CachedBeaconState<allForks.BeaconState>;

before(function () {
this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow

originalState = (generatePerfTestCachedStateAltair({
goBackOneSlot: true,
}) as unknown) as allForks.CachedBeaconState<allForks.BeaconState>;
const numPreviousEpochParticipation = originalState.previousEpochParticipation.persistent
.toArray()
.filter((part) => part && part.timelySource).length;
const numCurrentEpochParticipation = originalState.currentEpochParticipation.persistent
.toArray()
.filter((part) => part && part.timelySource).length;

expect(numPreviousEpochParticipation).to.equal(250000, "Wrong numPreviousEpochParticipation");
expect(numCurrentEpochParticipation).to.equal(250000, "Wrong numCurrentEpochParticipation");
});

itBench(
{
id: "getAttestationsForBlock",
beforeEach: () => {
const pool = new AggregatedAttestationPool();
for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) {
const slot = stateSlot - 1 - epochSlot;
const epoch = computeEpochAtSlot(slot);
const committeeCount = originalState.getCommitteeCountPerSlot(epoch);
const sourceCheckpoint = {
epoch: originalState.currentJustifiedCheckpoint.epoch,
root: originalState.currentJustifiedCheckpoint.root.valueOf() as Uint8Array,
};
for (let committeeIndex = 0; committeeIndex < committeeCount; committeeIndex++) {
const attestation = {
aggregationBits: Array.from({length: 64}, () => false) as List<boolean>,
data: {
slot: slot,
index: committeeIndex,
beaconBlockRoot: getBlockRootAtSlot(originalState, slot),
source: sourceCheckpoint,
target: {
epoch,
root: getBlockRootAtSlot(originalState, computeStartSlotAtEpoch(epoch)),
},
},
signature: Buffer.alloc(96),
};

const committee = originalState.getBeaconCommittee(slot, committeeIndex);
// all attestation has full participation so getAttestationsForBlock() has to do a lot of filter
pool.add(attestation, committee, committee);
}
}
return pool;
},
beforeEach: () => getAggregatedAttestationPool(originalState),
},
(pool) => {
// logger.info("Number of attestations in pool", pool.getAll().length);
pool.getAttestationsForBlock(originalState);
}
);
});

function getAggregatedAttestationPool(
state: allForks.CachedBeaconState<allForks.BeaconState>
): AggregatedAttestationPool {
const pool = new AggregatedAttestationPool();
for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) {
const slot = state.slot - 1 - epochSlot;
const epoch = computeEpochAtSlot(slot);
const committeeCount = state.getCommitteeCountPerSlot(epoch);
const sourceCheckpoint = {
epoch: state.currentJustifiedCheckpoint.epoch,
root: state.currentJustifiedCheckpoint.root.valueOf() as Uint8Array,
};
for (let committeeIndex = 0; committeeIndex < committeeCount; committeeIndex++) {
const attestation = {
aggregationBits: Array.from({length: 64}, () => false) as List<boolean>,
data: {
slot: slot,
index: committeeIndex,
beaconBlockRoot: getBlockRootAtSlot(state, slot),
source: sourceCheckpoint,
target: {
epoch,
root: getBlockRootAtSlot(state, computeStartSlotAtEpoch(epoch)),
},
},
signature: Buffer.alloc(96),
};

const committee = state.getBeaconCommittee(slot, committeeIndex);
// all attestation has full participation so getAttestationsForBlock() has to do a lot of filter
pool.add(attestation, committee, committee);
}
}
return pool;
}
File renamed without changes.
4 changes: 4 additions & 0 deletions packages/lodestar/test/setupPreset.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// Set minimal
if (process.env.LODESTAR_PRESET === undefined) {
process.env.LODESTAR_PRESET = "minimal";
}
4 changes: 2 additions & 2 deletions packages/lodestar/test/utils/validationData/attestation.ts
Original file line number Diff line number Diff line change
@@ -40,8 +40,8 @@ export function getAttestationValidData(
): {chain: IBeaconChain; attestation: phase0.Attestation; subnet: number; validatorIndex: number} {
const currentSlot = opts.currentSlot ?? 100;
const attSlot = opts.attSlot ?? currentSlot;
const attIndex = opts.attIndex ?? 1;
const bitIndex = opts.bitIndex ?? 1;
const attIndex = opts.attIndex ?? 0;
const bitIndex = opts.bitIndex ?? 0;
const targetRoot = opts.targetRoot ?? ZERO_HASH;
const beaconBlockRoot = opts.beaconBlockRoot ?? ZERO_HASH;
// Create cached state

1 comment on commit 485ec69

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for some benchmarks.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold.

Benchmark suite Current: 485ec69 Previous: 05e8265 Ratio
Process block - 250000 vs - 7PWei - with 0 validator exit 13.147 ms/op 354.94 us/op 37.04
Process block - 250000 vs - 7PWei - with 1 validator exit 118.28 ms/op 32.650 ms/op 3.62
shuffle list - 16384 els 37.375 ms/op 1.5060 ms/op 24.82
shuffle list - 250000 els 529.31 ms/op 21.502 ms/op 24.62
Full benchmark results
Benchmark suite Current: 485ec69 Previous: 05e8265 Ratio
getCommitteeAssignments - req 1000 vs - 250000 vc 9.2282 ms/op 7.8467 ms/op 1.18
epoch altair - 250000 vs - 7PWei - processInactivityUpdates 3.0492 s/op 2.2111 s/op 1.38
epoch altair - 250000 vs - 7PWei - processRewardsAndPenalties 1.2325 s/op 883.38 ms/op 1.40
epoch altair - 250000 vs - 7PWei - processParticipationFlagUpdates 572.53 ms/op 353.07 ms/op 1.62
Process block - 250000 vs - 7PWei - with 0 validator exit 13.147 ms/op 354.94 us/op 37.04
Process block - 250000 vs - 7PWei - with 1 validator exit 118.28 ms/op 32.650 ms/op 3.62
Process block - 250000 vs - 7PWei - with 16 validator exits 32.546 ms/op 26.815 ms/op 1.21
epoch phase0 - 250000 vs - 7PWei - beforeProcessEpoch 1.8831 s/op 634.02 ms/op 2.97
epoch phase0 - 250000 vs - 7PWei - processRewardsAndPenalties 954.61 ms/op 458.47 ms/op 2.08
epoch phase0 - 250000 vs - 7PWei - processEffectiveBalanceUpdates 188.89 ms/op 109.10 ms/op 1.73
getAttestationDeltas - 250000 vs - 7PWei 136.41 ms/op 92.590 ms/op 1.47
processSlots - 250000 vs - 7PWei - 32 empty slots 3.8874 s/op 4.5431 s/op 0.86
shuffle list - 16384 els 37.375 ms/op 1.5060 ms/op 24.82
shuffle list - 250000 els 529.31 ms/op 21.502 ms/op 24.62
getPubkeys - persistent - req 1000 vs - 250000 vc 22.626 us/op 15.169 us/op 1.49
BLS verify - blst-native 2.3864 ms/op 1.6439 ms/op 1.45
BLS verifyMultipleSignatures 3 - blst-native 4.8731 ms/op 3.3547 ms/op 1.45
BLS verifyMultipleSignatures 8 - blst-native 10.422 ms/op 7.2510 ms/op 1.44
BLS verifyMultipleSignatures 32 - blst-native 37.910 ms/op 28.700 ms/op 1.32
BLS aggregatePubkeys 32 - blst-native 54.187 us/op 35.819 us/op 1.51
BLS aggregatePubkeys 128 - blst-native 200.91 us/op 137.18 us/op 1.46
getAttestationsForBlock 146.81 ms/op 86.567 ms/op 1.70
validate gossip signedAggregateAndProof - struct 7.6318 ms/op 5.3061 ms/op 1.44
validate gossip signedAggregateAndProof - treeBacked 5.9595 ms/op 4.0418 ms/op 1.47
validate gossip attestation - struct 2.7016 ms/op 1.8708 ms/op 1.44
validate gossip attestation - treeBacked 2.8023 ms/op 1.9100 ms/op 1.47

Please sign in to comment.