diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 6c42228b5356..749fea1b82ff 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -4,11 +4,11 @@ import { IShufflingCache, ShufflingBuildProps, computeEpochShuffling, + computeEpochShufflingAsync, } from "@lodestar/state-transition"; import {Epoch, RootHex} from "@lodestar/types"; import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; -import {callInNextEventLoop} from "../util/eventLoop.js"; /** * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case: @@ -178,14 +178,19 @@ export class ShufflingCache implements IShufflingCache { this.insertPromise(epoch, decisionRoot); /** * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations - * on a NICE thread with a rust implementation + * on a NICE thread */ - callInNextEventLoop(() => { - const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "build"}); - const shuffling = computeEpochShuffling(state, activeIndices, epoch); - timer?.(); - this.set(shuffling, decisionRoot); - }); + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "build"}); + computeEpochShufflingAsync(state, activeIndices, epoch) + .then((shuffling) => { + this.set(shuffling, decisionRoot); + }) + .catch((err) => + this.logger?.error(`error building shuffling for epoch ${epoch} at decisionRoot ${decisionRoot}`, {}, err) + ) + .finally(() => { + timer?.(); + }); } /** diff --git a/packages/beacon-node/test/spec/presets/shuffling.test.ts b/packages/beacon-node/test/spec/presets/shuffling.test.ts index 06e7d4717d06..7b0f38ecf581 100644 --- a/packages/beacon-node/test/spec/presets/shuffling.test.ts +++ b/packages/beacon-node/test/spec/presets/shuffling.test.ts @@ -1,24 +1,27 @@ import path from "node:path"; -import {unshuffleList} from "@lodestar/state-transition"; +import {unshuffleList} from "@chainsafe/swap-or-not-shuffle"; import {InputType} from "@lodestar/spec-test-util"; import {bnToNum, fromHex} from "@lodestar/utils"; -import {ACTIVE_PRESET} from "@lodestar/params"; +import {ACTIVE_PRESET, SHUFFLE_ROUND_COUNT} from "@lodestar/params"; import {RunnerType, TestRunnerFn} from "../utils/types.js"; import {ethereumConsensusSpecsTests} from "../specTestVersioning.js"; import {specTestIterator} from "../utils/specTestIterator.js"; -const shuffling: TestRunnerFn = () => { +const shuffling: TestRunnerFn = () => { return { testFunction: (testcase) => { const seed = fromHex(testcase.mapping.seed); - const output = Array.from({length: bnToNum(testcase.mapping.count)}, (_, i) => i); - unshuffleList(output, seed); - return output; + const output = unshuffleList( + Uint32Array.from(Array.from({length: bnToNum(testcase.mapping.count)}, (_, i) => i)), + seed, + SHUFFLE_ROUND_COUNT + ); + return Buffer.from(output).toString("hex"); }, options: { inputTypes: {mapping: InputType.YAML}, timeout: 10000, - getExpected: (testCase) => testCase.mapping.mapping.map((value) => bnToNum(value)), + getExpected: (testCase) => Buffer.from(testCase.mapping.mapping.map((value) => bnToNum(value))).toString("hex"), // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts }, }; diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index e743ece013d9..a01d835bae95 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -63,6 +63,7 @@ "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.17.1", + "@chainsafe/swap-or-not-shuffle": "^0.0.2", "@lodestar/config": "^1.22.0", "@lodestar/params": "^1.22.0", "@lodestar/types": "^1.22.0", diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index c26c62fd2079..6f63a2f4f5f8 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -1,3 +1,4 @@ +import {asyncUnshuffleList, unshuffleList} from "@chainsafe/swap-or-not-shuffle"; import {Epoch, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; import {GaugeExtra, intDiv, Logger, NoLabels, toRootHex} from "@lodestar/utils"; import { @@ -6,11 +7,11 @@ import { MAX_COMMITTEES_PER_SLOT, SLOTS_PER_EPOCH, TARGET_COMMITTEE_SIZE, + SHUFFLE_ROUND_COUNT, } from "@lodestar/params"; import {BeaconConfig} from "@lodestar/config"; import {BeaconStateAllForks} from "../types.js"; import {getSeed} from "./seed.js"; -import {unshuffleList} from "./shuffle.js"; import {computeStartSlotAtEpoch} from "./epoch.js"; import {getBlockRootAtSlot} from "./blockRoot.js"; import {computeAnchorCheckpoint} from "./computeAnchorCheckpoint.js"; @@ -102,24 +103,15 @@ export function computeCommitteeCount(activeValidatorCount: number): number { return Math.max(1, Math.min(MAX_COMMITTEES_PER_SLOT, committeesPerSlot)); } -export function computeEpochShuffling( - state: BeaconStateAllForks, - activeIndices: Uint32Array, - epoch: Epoch -): EpochShuffling { - const activeValidatorCount = activeIndices.length; - - const shuffling = activeIndices.slice(); - const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); - unshuffleList(shuffling, seed); - +function buildCommitteesFromShuffling(shuffling: Uint32Array): Uint32Array[][] { + const activeValidatorCount = shuffling.length; const committeesPerSlot = computeCommitteeCount(activeValidatorCount); - const committeeCount = committeesPerSlot * SLOTS_PER_EPOCH; - const committees: Uint32Array[][] = []; + const committees = new Array(SLOTS_PER_EPOCH); for (let slot = 0; slot < SLOTS_PER_EPOCH; slot++) { - const slotCommittees: Uint32Array[] = []; + const slotCommittees = new Array(committeesPerSlot); + for (let committeeIndex = 0; committeeIndex < committeesPerSlot; committeeIndex++) { const index = slot * committeesPerSlot + committeeIndex; const startOffset = Math.floor((activeValidatorCount * index) / committeeCount); @@ -127,17 +119,48 @@ export function computeEpochShuffling( if (!(startOffset <= endOffset)) { throw new Error(`Invalid offsets: start ${startOffset} must be less than or equal end ${endOffset}`); } - slotCommittees.push(shuffling.subarray(startOffset, endOffset)); + slotCommittees[committeeIndex] = shuffling.subarray(startOffset, endOffset); } - committees.push(slotCommittees); + + committees[slot] = slotCommittees; } + return committees; +} + +export function computeEpochShuffling( + // TODO: (@matthewkeil) remove state/epoch and pass in seed to clean this up + state: BeaconStateAllForks, + activeIndices: Uint32Array, + epoch: Epoch +): EpochShuffling { + const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); + const shuffling = unshuffleList(activeIndices, seed, SHUFFLE_ROUND_COUNT); + const committees = buildCommitteesFromShuffling(shuffling); + return { + epoch, + activeIndices, + shuffling, + committees, + committeesPerSlot: committees[0].length, + }; +} + +export async function computeEpochShufflingAsync( + // TODO: (@matthewkeil) remove state/epoch and pass in seed to clean this up + state: BeaconStateAllForks, + activeIndices: Uint32Array, + epoch: Epoch +): Promise { + const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); + const shuffling = await asyncUnshuffleList(activeIndices, seed, SHUFFLE_ROUND_COUNT); + const committees = buildCommitteesFromShuffling(shuffling); return { epoch, activeIndices, shuffling, committees, - committeesPerSlot, + committeesPerSlot: committees[0].length, }; } diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index 5f8d9e5cdcfc..b88a20719b85 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -17,7 +17,6 @@ export * from "./genesis.js"; export * from "./interop.js"; export * from "./rootCache.js"; export * from "./seed.js"; -export * from "./shuffle.js"; export * from "./shufflingDecisionRoot.js"; export * from "./signatureSets.js"; export * from "./signingRoot.js"; diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts deleted file mode 100644 index a87f22cae43e..000000000000 --- a/packages/state-transition/src/util/shuffle.ts +++ /dev/null @@ -1,221 +0,0 @@ -import {digest} from "@chainsafe/as-sha256"; -import {SHUFFLE_ROUND_COUNT} from "@lodestar/params"; -import {Bytes32} from "@lodestar/types"; -import {assert, bytesToBigInt} from "@lodestar/utils"; - -// ArrayLike but with settable indices -type Shuffleable = { - readonly length: number; - [index: number]: number; -}; - -// ShuffleList shuffles a list, using the given seed for randomness. Mutates the input list. -export function shuffleList(input: Shuffleable, seed: Bytes32): void { - innerShuffleList(input, seed, true); -} - -// UnshuffleList undoes a list shuffling using the seed of the shuffling. Mutates the input list. -export function unshuffleList(input: Shuffleable, seed: Bytes32): void { - innerShuffleList(input, seed, false); -} - -const _SHUFFLE_H_SEED_SIZE = 32; -const _SHUFFLE_H_ROUND_SIZE = 1; -const _SHUFFLE_H_POSITION_WINDOW_SIZE = 4; -const _SHUFFLE_H_PIVOT_VIEW_SIZE = _SHUFFLE_H_SEED_SIZE + _SHUFFLE_H_ROUND_SIZE; -const _SHUFFLE_H_TOTAL_SIZE = _SHUFFLE_H_SEED_SIZE + _SHUFFLE_H_ROUND_SIZE + _SHUFFLE_H_POSITION_WINDOW_SIZE; - -/* - -def shuffle(list_size, seed): - indices = list(range(list_size)) - for round in range(90): - hash_bytes = b''.join([ - hash(seed + round.to_bytes(1, 'little') + (i).to_bytes(4, 'little')) - for i in range((list_size + 255) // 256) - ]) - pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'little')), 'little') % list_size - - powers_of_two = [1, 2, 4, 8, 16, 32, 64, 128] - - for i, index in enumerate(indices): - flip = (pivot - index) % list_size - hash_pos = index if index > flip else flip - byte = hash_bytes[hash_pos // 8] - if byte & powers_of_two[hash_pos % 8]: - indices[i] = flip - return indices - -Heavily-optimized version of the set-shuffling algorithm proposed by Vitalik to shuffle all items in a list together. - -Original here: - https://github.com/ethereum/consensus-specs/pull/576 - -Main differences, implemented by @protolambda: - - User can supply input slice to shuffle, simple provide [0,1,2,3,4, ...] to get a list of cleanly shuffled indices. - - Input slice is shuffled (hence no return value), no new array is allocated - - Allocations as minimal as possible: only a very minimal buffer for hashing - (this should be allocated on the stack, compiler will find it with escape analysis). - This is not bigger than what's used for shuffling a single index! - As opposed to larger allocations (size O(n) instead of O(1)) made in the original. - - Replaced pseudocode/python workarounds with bit-logic. - - User can provide their own hash-function (as long as it outputs a 32 len byte slice) - -This Typescript version is an adaption of the Python version, in turn an adaption of the original Go version. -Python: https://github.com/protolambda/eth2fastspec/blob/14e04e9db77ef7c8b7788ffdaa7e142d7318dd7e/eth2fastspec.py#L63 -Go: https://github.com/protolambda/eth2-shuffle -All three implemented by @protolambda, but meant for public use, like the original spec version. -*/ - -function setPositionUint32(value: number, buf: Buffer): void { - // Little endian, optimized version - buf[_SHUFFLE_H_PIVOT_VIEW_SIZE] = (value >> 0) & 0xff; - buf[_SHUFFLE_H_PIVOT_VIEW_SIZE + 1] = (value >> 8) & 0xff; - buf[_SHUFFLE_H_PIVOT_VIEW_SIZE + 2] = (value >> 16) & 0xff; - buf[_SHUFFLE_H_PIVOT_VIEW_SIZE + 3] = (value >> 24) & 0xff; -} - -// Shuffles or unshuffles, depending on the `dir` (true for shuffling, false for unshuffling -function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void { - if (input.length <= 1) { - // nothing to (un)shuffle - return; - } - if (SHUFFLE_ROUND_COUNT == 0) { - // no shuffling - return; - } - // uint32 is sufficient, and necessary in JS, - // as we do a lot of bit math on it, which cannot be done as fast on more bits. - const listSize = input.length >>> 0; - // check if list size fits in uint32 - assert.equal(listSize, input.length, "input length does not fit uint32"); - // check that the seed is 32 bytes - assert.lte(seed.length, _SHUFFLE_H_SEED_SIZE, `seed length is not lte ${_SHUFFLE_H_SEED_SIZE} bytes`); - - const buf = Buffer.alloc(_SHUFFLE_H_TOTAL_SIZE); - let r = 0; - if (!dir) { - // Start at last round. - // Iterating through the rounds in reverse, un-swaps everything, effectively un-shuffling the list. - r = SHUFFLE_ROUND_COUNT - 1; - } - - // Seed is always the first 32 bytes of the hash input, we never have to change this part of the buffer. - buf.set(seed, 0); - - // initial values here are not used: overwritten first within the inner for loop. - let source = seed; // just setting it to a Bytes32 - let byteV = 0; - - while (true) { - // spec: pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size - // This is the "int_to_bytes1(round)", appended to the seed. - buf[_SHUFFLE_H_SEED_SIZE] = r; - // Seed is already in place, now just hash the correct part of the buffer, and take a uint64 from it, - // and modulo it to get a pivot within range. - const h = digest(buf.subarray(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); - const pivot = Number(bytesToBigInt(h.subarray(0, 8)) % BigInt(listSize)) >>> 0; - - // Split up the for-loop in two: - // 1. Handle the part from 0 (incl) to pivot (incl). This is mirrored around (pivot / 2) - // 2. Handle the part from pivot (excl) to N (excl). This is mirrored around ((pivot / 2) + (size/2)) - // The pivot defines a split in the array, with each of the splits mirroring their data within the split. - // Print out some example even/odd sized index lists, with some even/odd pivots, - // and you can deduce how the mirroring works exactly. - // Note that the mirror is strict enough to not consider swapping the index @mirror with itself. - let mirror = (pivot + 1) >> 1; - // Since we are iterating through the "positions" in order, we can just repeat the hash every 256th position. - // No need to pre-compute every possible hash for efficiency like in the example code. - // We only need it consecutively (we are going through each in reverse order however, but same thing) - // - // spec: source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) - // - seed is still in 0:32 (excl., 32 bytes) - // - round number is still in 32 - // - mix in the position for randomness, except the last byte of it, - // which will be used later to select a bit from the resulting hash. - // We start from the pivot position, and work back to the mirror position (of the part left to the pivot). - // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) - setPositionUint32(pivot >> 8, buf); // already using first pivot byte below. - source = digest(buf); - byteV = source[(pivot & 0xff) >> 3]; - - for (let i = 0, j; i < mirror; i++) { - j = pivot - i; - // -- step() fn start - // The pair is i,j. With j being the bigger of the two, hence the "position" identifier of the pair. - // Every 256th bit (aligned to j). - if ((j & 0xff) == 0xff) { - // just overwrite the last part of the buffer, reuse the start (seed, round) - setPositionUint32(j >> 8, buf); - source = digest(buf); - } - - // Same trick with byte retrieval. Only every 8th. - if ((j & 0x7) == 0x7) { - byteV = source[(j & 0xff) >> 3]; - } - - const bitV = (byteV >> (j & 0x7)) & 0x1; - - if (bitV == 1) { - // swap the pair items - const tmp = input[j]; - input[j] = input[i]; - input[i] = tmp; - } - // -- step() fn end - } - - // Now repeat, but for the part after the pivot. - mirror = (pivot + listSize + 1) >> 1; - const end = listSize - 1; - // Again, seed and round input is in place, just update the position. - // We start at the end, and work back to the mirror point. - // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) - setPositionUint32(end >> 8, buf); - source = digest(buf); - byteV = source[(end & 0xff) >> 3]; - for (let i = pivot + 1, j; i < mirror; i++) { - j = end - i + pivot + 1; - // -- step() fn start - // The pair is i,j. With j being the bigger of the two, hence the "position" identifier of the pair. - // Every 256th bit (aligned to j). - if ((j & 0xff) == 0xff) { - // just overwrite the last part of the buffer, reuse the start (seed, round) - setPositionUint32(j >> 8, buf); - source = digest(buf); - } - - // Same trick with byte retrieval. Only every 8th. - if ((j & 0x7) == 0x7) { - byteV = source[(j & 0xff) >> 3]; - } - - const bitV = (byteV >> (j & 0x7)) & 0x1; - - if (bitV == 1) { - // swap the pair items - const tmp = input[j]; - input[j] = input[i]; - input[i] = tmp; - } - // -- step() fn end - } - - // go forwards? - if (dir) { - // -> shuffle - r += 1; - if (r == SHUFFLE_ROUND_COUNT) { - break; - } - } else { - if (r == 0) { - break; - } - // -> un-shuffle - r -= 1; - } - } -} diff --git a/packages/state-transition/test/perf/hashing.test.ts b/packages/state-transition/test/perf/hashing.test.ts index b7afc59717bc..26bfd935c08a 100644 --- a/packages/state-transition/test/perf/hashing.test.ts +++ b/packages/state-transition/test/perf/hashing.test.ts @@ -1,13 +1,14 @@ import {itBench} from "@dapplion/benchmark"; +import {unshuffleList} from "@chainsafe/swap-or-not-shuffle"; import {ssz} from "@lodestar/types"; -import {unshuffleList} from "../../src/index.js"; +import {SHUFFLE_ROUND_COUNT} from "@lodestar/params"; import {generatePerfTestCachedStatePhase0, numValidators} from "./util.js"; // Test cost of hashing state after some modifications describe("BeaconState hashTreeRoot", () => { const vc = numValidators; - const indicesShuffled: number[] = []; + let indicesShuffled: Uint32Array; let stateOg: ReturnType; before(function () { @@ -15,8 +16,13 @@ describe("BeaconState hashTreeRoot", () => { stateOg = generatePerfTestCachedStatePhase0(); stateOg.hashTreeRoot(); - for (let i = 0; i < vc; i++) indicesShuffled[i] = i; - unshuffleList(indicesShuffled, new Uint8Array([42, 32])); + const seed = new Uint8Array(32); + seed.set([42, 32], 0); + const preShuffle = new Uint32Array(numValidators); + for (let i = 0; i < vc; i++) { + preShuffle[i] = i; + } + indicesShuffled = unshuffleList(preShuffle, seed, SHUFFLE_ROUND_COUNT); }); const validator = ssz.phase0.Validator.defaultViewDU(); diff --git a/packages/state-transition/test/perf/shuffle/numberMath.test.ts b/packages/state-transition/test/perf/shuffle/numberMath.test.ts deleted file mode 100644 index 08e57b090aa3..000000000000 --- a/packages/state-transition/test/perf/shuffle/numberMath.test.ts +++ /dev/null @@ -1,82 +0,0 @@ -import {itBench} from "@dapplion/benchmark"; - -describe.skip("shuffle number math ops", () => { - const forRuns = 100e5; - const j = forRuns / 2; - - const arrSize = forRuns; - const input: number[] = []; - const inputUint32Array = new Uint32Array(arrSize); - for (let i = 0; i < arrSize; i++) { - input[i] = i; - inputUint32Array[i] = i; - } - - itBench("if(i == 1)", () => { - for (let i = 0; i < forRuns; i++) { - if (i === 1) { - // - } - } - }); - - itBench("if(i)", () => { - for (let i = 0; i < forRuns; i++) { - if (i) { - // - } - } - }); - - itBench("i == j", () => { - for (let i = 0; i < forRuns; i++) { - i == j; - } - }); - - itBench("i === j", () => { - for (let i = 0; i < forRuns; i++) { - i === j; - } - }); - - itBench("bit opts", () => { - for (let i = 0; i < forRuns; i++) { - (j & 0x7) == 0x7; - } - }); - - itBench("modulo", () => { - for (let i = 0; i < forRuns; i++) { - j % 8 == 0; - } - }); - - itBench(">> 3", () => { - for (let i = 0; i < forRuns; i++) { - j >> 3; - } - }); - - itBench("/ 8", () => { - for (let i = 0; i < forRuns; i++) { - j / 8; - } - }); - - itBench("swap item in array", () => { - for (let i = 0; i < forRuns; i++) { - const tmp = input[forRuns - i]; - input[forRuns - i] = input[i]; - input[i] = tmp; - } - }); - - itBench("swap item in Uint32Array", () => { - for (let i = 0; i < forRuns; i++) { - const tmp = inputUint32Array[forRuns - i]; - inputUint32Array[forRuns - i] = inputUint32Array[i]; - inputUint32Array[i] = tmp; - } - }); -}); diff --git a/packages/state-transition/test/perf/shuffle/shuffle.test.ts b/packages/state-transition/test/perf/shuffle/shuffle.test.ts deleted file mode 100644 index 55f7875e69dd..000000000000 --- a/packages/state-transition/test/perf/shuffle/shuffle.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {itBench} from "@dapplion/benchmark"; -import {unshuffleList} from "../../../src/index.js"; - -// Lightouse Lodestar -// 512 254.04 us 1.6034 ms (x6) -// 16384 6.2046 ms 18.272 ms (x3) -// 4000000 1.5617 s 4.9690 s (x3) - -describe("shuffle list", () => { - const seed = new Uint8Array([42, 32]); - - for (const listSize of [ - 16384, - 250000, - // Don't run 4_000_000 since it's very slow and not testnet has gotten there yet - // 4e6, - ]) { - itBench({ - id: `shuffle list - ${listSize} els`, - before: () => { - const input: number[] = []; - for (let i = 0; i < listSize; i++) input[i] = i; - return new Uint32Array(input); - }, - beforeEach: (input) => input, - fn: (input) => unshuffleList(input, seed), - }); - } -}); diff --git a/packages/state-transition/test/unit/util/shuffle.test.ts b/packages/state-transition/test/unit/util/shuffle.test.ts deleted file mode 100644 index 9186968674ae..000000000000 --- a/packages/state-transition/test/unit/util/shuffle.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {expect, describe, it} from "vitest"; -import {unshuffleList} from "../../../src/index.js"; - -describe("util / shuffle", () => { - const testCases: { - id: string; - input: number[]; - res: number[]; - }[] = [ - // Values from `unshuffleList()` at commit https://github.com/ChainSafe/lodestar/commit/ec065635ca7da7f3788da018bd68c4900f0427d2 - { - id: "8 elements", - input: [0, 1, 2, 3, 4, 5, 6, 7], - res: [6, 3, 4, 0, 1, 5, 7, 2], - }, - { - id: "16 elements", - input: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - res: [8, 4, 11, 7, 10, 6, 0, 3, 15, 12, 5, 14, 1, 9, 13, 2], - }, - ]; - - const seed = new Uint8Array([42, 32]); - - it.each(testCases)("$id", ({input, res}) => { - unshuffleList(input, seed); - expect(input).toEqual(res); - }); -}); diff --git a/packages/state-transition/test/unit/util/shuffling.test.ts b/packages/state-transition/test/unit/util/shuffling.test.ts new file mode 100644 index 000000000000..f4039b472f5c --- /dev/null +++ b/packages/state-transition/test/unit/util/shuffling.test.ts @@ -0,0 +1,32 @@ +import {describe, it, expect} from "vitest"; +import {ssz} from "@lodestar/types"; +import {generateState} from "../../utils/state.js"; +import {computeEpochShuffling, computeEpochShufflingAsync} from "../../../src/util/epochShuffling.js"; +import {computeEpochAtSlot} from "../../../src/index.js"; + +describe("EpochShuffling", () => { + it("async and sync versions should be identical", async () => { + const numberOfValidators = 1000; + const activeIndices = Uint32Array.from(Array.from({length: numberOfValidators}, (_, i) => i)); + const state = generateState(); + state.slot = 12345; + state.validators = ssz.phase0.Validators.toViewDU( + Array.from({length: numberOfValidators}, () => ({ + activationEligibilityEpoch: 0, + activationEpoch: 0, + exitEpoch: Infinity, + effectiveBalance: 32, + pubkey: Buffer.alloc(48, 0xaa), + slashed: false, + withdrawableEpoch: Infinity, + withdrawalCredentials: Buffer.alloc(8, 0x01), + })) + ); + const epoch = computeEpochAtSlot(state.slot); + + const sync = computeEpochShuffling(state, activeIndices, epoch); + const async = await computeEpochShufflingAsync(state, activeIndices, epoch); + + expect(sync).toStrictEqual(async); + }); +}); diff --git a/yarn.lock b/yarn.lock index 9808fee96b84..3907e9f4e441 100644 --- a/yarn.lock +++ b/yarn.lock @@ -674,6 +674,60 @@ "@chainsafe/as-sha256" "0.5.0" "@chainsafe/persistent-merkle-tree" "0.8.0" +"@chainsafe/swap-or-not-shuffle-darwin-arm64@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-darwin-arm64/-/swap-or-not-shuffle-darwin-arm64-0.0.2.tgz#9c4b02970619b9ec5274f357f3a03ff97e745957" + integrity sha512-e2tmpSgGTHFv1g3oEKP/YElDTmxZwuSwVQ+Cf0gTUA8z4YQsJar61293My6JeA7qC5razWjhvcEk13ZbTCqk0w== + +"@chainsafe/swap-or-not-shuffle-darwin-x64@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-darwin-x64/-/swap-or-not-shuffle-darwin-x64-0.0.2.tgz#d2777b93455a217bf4b5b94b84e76239bc3ce860" + integrity sha512-JrK4psAQz0HzlfnsGRcHnhJSv6OHfjVNkEtERnpAjyeigwCRh09wNBzHEnIk8SOGFk29DabI4FC1wdh2Cs0DAA== + +"@chainsafe/swap-or-not-shuffle-linux-arm64-gnu@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-linux-arm64-gnu/-/swap-or-not-shuffle-linux-arm64-gnu-0.0.2.tgz#a881c6e29704bbbaceb8b75fd9394832087979f3" + integrity sha512-nGTIRUXt1QRNiWQXZnA8IWiHnAw6FNkU7RkngkoDzjD8pEhrtWs8tv/pdOxRKEhw21HBvKi7z8J+a7/MtxDLTg== + +"@chainsafe/swap-or-not-shuffle-linux-arm64-musl@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-linux-arm64-musl/-/swap-or-not-shuffle-linux-arm64-musl-0.0.2.tgz#118f80f7bd7e8f83fb566112a17003f228d86593" + integrity sha512-sumwkxQ0Mky+W66Jf43cHUybgHQ4FENj2iRBRw3jGWiZ79Vv/DZ1dMA6I4/LVWCsvZmFUIvMvKNthGHXefh2DQ== + +"@chainsafe/swap-or-not-shuffle-linux-x64-gnu@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-linux-x64-gnu/-/swap-or-not-shuffle-linux-x64-gnu-0.0.2.tgz#2dca41ba2848f904b5c9bc831cb0428c958261f3" + integrity sha512-do9NH/43eUWxtY+k3fxFltSSfKJpyvAINA/dJ3EHVkqS/oBTwR450CVNV5HFcR5Uvgxuth3/BjVYb4APs8N/MQ== + +"@chainsafe/swap-or-not-shuffle-linux-x64-musl@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-linux-x64-musl/-/swap-or-not-shuffle-linux-x64-musl-0.0.2.tgz#75025eee02e2b34f10852c1de57b8e0050584320" + integrity sha512-ClwDMZd768PIaAUQhbyZpqqip0b6Sgjt8IpP5ACYMJr2AHoiG64POZaiFu+zusT4q3s4/dvqaz86jRQaF4vFkw== + +"@chainsafe/swap-or-not-shuffle-win32-arm64-msvc@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-win32-arm64-msvc/-/swap-or-not-shuffle-win32-arm64-msvc-0.0.2.tgz#f5520df2bf72e823ef225d9f13d317b91b6372e4" + integrity sha512-i9+qj0VSppy2xMChQE38rCmWmmy8SJ0uSaApc0L4KZ+t2aqquYyEUXWGgfdXMZx9MqAUvuigzv34T9qivADFag== + +"@chainsafe/swap-or-not-shuffle-win32-x64-msvc@0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle-win32-x64-msvc/-/swap-or-not-shuffle-win32-x64-msvc-0.0.2.tgz#7413aa32a4006bf144b66087844aee71bb2a5f42" + integrity sha512-zZ9J0PWzGEgfjjAVMeTvAPdHNs5XCeQqfNQ0E/lQUYhS8Wi84y44R+7M6C/KIcS0GmvCpZXI2sQGeNjqbU5LoA== + +"@chainsafe/swap-or-not-shuffle@^0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/swap-or-not-shuffle/-/swap-or-not-shuffle-0.0.2.tgz#59a148b80bb8e8d4f2f38f7bd00562c7a6b99e62" + integrity sha512-Rbk3p86cX71VAq46+nRcBOHOetK2ls4QrAUf1YOWTYLHmmLvx8j9Q6gQndjtu0OeYGd7Eyj1pQCFDlS9kixk+g== + optionalDependencies: + "@chainsafe/swap-or-not-shuffle-darwin-arm64" "0.0.2" + "@chainsafe/swap-or-not-shuffle-darwin-x64" "0.0.2" + "@chainsafe/swap-or-not-shuffle-linux-arm64-gnu" "0.0.2" + "@chainsafe/swap-or-not-shuffle-linux-arm64-musl" "0.0.2" + "@chainsafe/swap-or-not-shuffle-linux-x64-gnu" "0.0.2" + "@chainsafe/swap-or-not-shuffle-linux-x64-musl" "0.0.2" + "@chainsafe/swap-or-not-shuffle-win32-arm64-msvc" "0.0.2" + "@chainsafe/swap-or-not-shuffle-win32-x64-msvc" "0.0.2" + "@chainsafe/threads@^1.11.1": version "1.11.1" resolved "https://registry.yarnpkg.com/@chainsafe/threads/-/threads-1.11.1.tgz#0b3b8c76f5875043ef6d47aeeb681dc80378f205"