feat: improve before process epoch

packages/state-transition/src/cache/epochCache.ts

@@ -338,11 +338,16 @@ export class EpochCache {
       throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low.");
     }
 
-    const currentShuffling = cachedCurrentShuffling ?? computeEpochShuffling(state, currentActiveIndices, currentEpoch);
+    const currentShuffling =
+      cachedCurrentShuffling ??
+      computeEpochShuffling(state, currentActiveIndices, currentActiveIndices.length, currentEpoch);
     const previousShuffling =
       cachedPreviousShuffling ??
-      (isGenesis ? currentShuffling : computeEpochShuffling(state, previousActiveIndices, previousEpoch));
-    const nextShuffling = cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextEpoch);
+      (isGenesis
+        ? currentShuffling
+        : computeEpochShuffling(state, previousActiveIndices, previousActiveIndices.length, previousEpoch));
+    const nextShuffling =
+      cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch);
 
     const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER);
 
@@ -501,6 +506,7 @@ export class EpochCache {
     state: BeaconStateAllForks,
     epochTransitionCache: {
       nextEpochShufflingActiveValidatorIndices: ValidatorIndex[];
+      totalNextEpochShufflingActiveIndices: number;
       nextEpochTotalActiveBalanceByIncrement: number;
     }
   ): void {
@@ -512,6 +518,7 @@ export class EpochCache {
     this.nextShuffling = computeEpochShuffling(
       state,
       epochTransitionCache.nextEpochShufflingActiveValidatorIndices,
+      epochTransitionCache.totalNextEpochShufflingActiveIndices,
       nextEpoch
     );
 

packages/state-transition/src/cache/epochTransitionCache.ts

@@ -52,15 +52,6 @@ export interface EpochTransitionCache {
   };
   currEpochUnslashedTargetStakeByIncrement: number;
 
-  /**
-   * Validator indices that are either
-   * - active in previous epoch
-   * - slashed and not yet withdrawable
-   *
-   * getRewardsAndPenalties() and processInactivityUpdates() iterate this list
-   */
-  eligibleValidatorIndices: ValidatorIndex[];
-
   /**
    * Indices which will receive the slashing penalty
    * ```
@@ -125,10 +116,13 @@ export interface EpochTransitionCache {
    * - un-slashed validators
    * - prev attester flag set
    * With a status flag to check this conditions at once we just have to mask with an OR of the conditions.
+   * This is only for phase0 only.
    */
-
   proposerIndices: number[];
 
+  /**
+   * This is for phase0 only.
+   */
   inclusionDelays: number[];
 
   flags: number[];
@@ -151,6 +145,11 @@ export interface EpochTransitionCache {
    */
   nextEpochShufflingActiveValidatorIndices: ValidatorIndex[];
 
+  /**
+   * We do not use up to `nextEpochShufflingActiveValidatorIndices.length`, use this to control that
+   */
+  totalNextEpochShufflingActiveIndices: number;
+
   /**
    * Altair specific, this is total active balances for the next epoch.
    * This is only used in `afterProcessEpoch` to compute base reward and sync participant reward.
@@ -191,12 +190,14 @@ const isActivePrevEpoch = new Array<boolean>();
 const isActiveCurrEpoch = new Array<boolean>();
 /** WARNING: reused, never gc'd */
 const isActiveNextEpoch = new Array<boolean>();
-/** WARNING: reused, never gc'd */
+/** WARNING: reused, never gc'd, from altair this is empty array */
 const proposerIndices = new Array<number>();
-/** WARNING: reused, never gc'd */
+/** WARNING: reused, never gc'd, from altair this is empty array */
 const inclusionDelays = new Array<number>();
 /** WARNING: reused, never gc'd */
 const flags = new Array<number>();
+/** WARNING: reused, never gc'd */
+const nextEpochShufflingActiveValidatorIndices = new Array<number>();
 
 export function beforeProcessEpoch(
   state: CachedBeaconStateAllForks,
@@ -212,12 +213,10 @@ export function beforeProcessEpoch(
 
   const slashingsEpoch = currentEpoch + intDiv(EPOCHS_PER_SLASHINGS_VECTOR, 2);
 
-  const eligibleValidatorIndices: ValidatorIndex[] = [];
   const indicesToSlash: ValidatorIndex[] = [];
   const indicesEligibleForActivationQueue: ValidatorIndex[] = [];
   const indicesEligibleForActivation: ValidatorIndex[] = [];
   const indicesToEject: ValidatorIndex[] = [];
-  const nextEpochShufflingActiveValidatorIndices: ValidatorIndex[] = [];
 
   let totalActiveStakeByIncrement = 0;
 
@@ -227,6 +226,8 @@ export function beforeProcessEpoch(
   const validators = state.validators.getAllReadonlyValues();
   const validatorCount = validators.length;
 
+  nextEpochShufflingActiveValidatorIndices.length = validatorCount;
+  let totalNextEpochShufflingActiveIndices = 0;
   // pre-fill with true (most validators are active)
   isActivePrevEpoch.length = validatorCount;
   isActiveCurrEpoch.length = validatorCount;
@@ -238,14 +239,10 @@ export function beforeProcessEpoch(
   // During the epoch transition, additional data is precomputed to avoid traversing any state a second
   // time. Attestations are a big part of this, and each validator has a "status" to represent its
   // precomputed participation.
-  // - proposerIndex: number; // -1 when not included by any proposer
-  // - inclusionDelay: number;
+  // - proposerIndex: number; // -1 when not included by any proposer, for phase0 only so it's declared inside phase0 block below
+  // - inclusionDelay: number;// for phase0 only so it's declared inside phase0 block below
   // - flags: number; // bitfield of AttesterFlags
-  proposerIndices.length = validatorCount;
-  inclusionDelays.length = validatorCount;
   flags.length = validatorCount;
-  proposerIndices.fill(-1);
-  inclusionDelays.fill(0);
   // flags.fill(0);
   // flags will be zero'd out below
   // In the first loop, set slashed+eligibility
@@ -284,7 +281,6 @@ export function beforeProcessEpoch(
     // This is done to prevent self-slashing from being a way to escape inactivity leaks.
     // TODO: Consider using an array of `eligibleValidatorIndices: number[]`
     if (isActivePrev || (validator.slashed && prevEpoch + 1 < validator.withdrawableEpoch)) {
-      eligibleValidatorIndices.push(i);
       flag |= FLAG_ELIGIBLE_ATTESTER;
     }
 
@@ -348,7 +344,7 @@ export function beforeProcessEpoch(
     }
 
     if (isActiveNext2) {
-      nextEpochShufflingActiveValidatorIndices.push(i);
+      nextEpochShufflingActiveValidatorIndices[totalNextEpochShufflingActiveIndices++] = i;
     }
   }
 
@@ -368,6 +364,10 @@ export function beforeProcessEpoch(
   );
 
   if (forkSeq === ForkSeq.phase0) {
+    proposerIndices.length = validatorCount;
+    proposerIndices.fill(-1);
+    inclusionDelays.length = validatorCount;
+    inclusionDelays.fill(0);
     processPendingAttestations(
       state as CachedBeaconStatePhase0,
       proposerIndices,
@@ -467,12 +467,12 @@ export function beforeProcessEpoch(
       headStakeByIncrement: prevHeadUnslStake,
     },
     currEpochUnslashedTargetStakeByIncrement: currTargetUnslStake,
-    eligibleValidatorIndices,
     indicesToSlash,
     indicesEligibleForActivationQueue,
     indicesEligibleForActivation,
     indicesToEject,
     nextEpochShufflingActiveValidatorIndices,
+    totalNextEpochShufflingActiveIndices,
     // to be updated in processEffectiveBalanceUpdates
     nextEpochTotalActiveBalanceByIncrement: 0,
     isActivePrevEpoch,

packages/state-transition/src/epoch/processInactivityUpdates.ts

@@ -24,30 +24,31 @@ export function processInactivityUpdates(state: CachedBeaconStateAltair, cache:
 
   const {config, inactivityScores} = state;
   const {INACTIVITY_SCORE_BIAS, INACTIVITY_SCORE_RECOVERY_RATE} = config;
-  const {flags, eligibleValidatorIndices} = cache;
+  const {flags} = cache;
   const inActivityLeak = isInInactivityLeak(state);
 
   // this avoids importing FLAG_ELIGIBLE_ATTESTER inside the for loop, check the compiled code
-  const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers} = attesterStatusUtil;
+  const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, FLAG_ELIGIBLE_ATTESTER, hasMarkers} = attesterStatusUtil;
 
   const inactivityScoresArr = inactivityScores.getAll();
 
-  for (let j = 0; j < eligibleValidatorIndices.length; j++) {
-    const i = eligibleValidatorIndices[j];
+  for (let i = 0; i < flags.length; i++) {
     const flag = flags[i];
-    let inactivityScore = inactivityScoresArr[i];
+    if (hasMarkers(flag, FLAG_ELIGIBLE_ATTESTER)) {
+      let inactivityScore = inactivityScoresArr[i];
 
-    const prevInactivityScore = inactivityScore;
-    if (hasMarkers(flag, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) {
-      inactivityScore -= Math.min(1, inactivityScore);
-    } else {
-      inactivityScore += INACTIVITY_SCORE_BIAS;
-    }
-    if (!inActivityLeak) {
-      inactivityScore -= Math.min(INACTIVITY_SCORE_RECOVERY_RATE, inactivityScore);
-    }
-    if (inactivityScore !== prevInactivityScore) {
-      inactivityScores.set(i, inactivityScore);
+      const prevInactivityScore = inactivityScore;
+      if (hasMarkers(flag, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) {
+        inactivityScore -= Math.min(1, inactivityScore);
+      } else {
+        inactivityScore += INACTIVITY_SCORE_BIAS;
+      }
+      if (!inActivityLeak) {
+        inactivityScore -= Math.min(INACTIVITY_SCORE_RECOVERY_RATE, inactivityScore);
+      }
+      if (inactivityScore !== prevInactivityScore) {
+        inactivityScores.set(i, inactivityScore);
+      }
     }
   }
 }

packages/state-transition/src/epoch/processRegistryUpdates.ts

@@ -37,9 +37,13 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache:
   }
 
   const finalityEpoch = state.finalizedCheckpoint.epoch;
+  // this avoids an array allocation compared to `slice(0, epochCtx.activationChurnLimit)`
+  const len = Math.min(cache.indicesEligibleForActivation.length, epochCtx.activationChurnLimit);
+  const activationEpoch = computeActivationExitEpoch(cache.currentEpoch);
   // dequeue validators for activation up to churn limit
-  for (const index of cache.indicesEligibleForActivation.slice(0, epochCtx.activationChurnLimit)) {
-    const validator = validators.get(index);
+  for (let i = 0; i < len; i++) {
+    const validatorIndex = cache.indicesEligibleForActivation[i];
+    const validator = validators.get(validatorIndex);
     // placement in queue is finalized
     if (validator.activationEligibilityEpoch > finalityEpoch) {
       // remaining validators all have an activationEligibilityEpoch that is higher anyway, break early
@@ -48,6 +52,6 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache:
       // So we need to filter by finalityEpoch here to comply with the spec.
       break;
     }
-    validator.activationEpoch = computeActivationExitEpoch(cache.currentEpoch);
+    validator.activationEpoch = activationEpoch;
   }
 }

packages/state-transition/src/util/epochShuffling.ts

@@ -62,16 +62,22 @@ export function computeCommitteeCount(activeValidatorCount: number): number {
 export function computeEpochShuffling(
   state: BeaconStateAllForks,
   activeIndices: ArrayLike<ValidatorIndex>,
+  activeValidatorCount: number,
   epoch: Epoch
 ): EpochShuffling {
   const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER);
 
-  // copy
-  const _activeIndices = new Uint32Array(activeIndices);
+  if (activeValidatorCount > activeIndices.length) {
+    throw new Error(`Invalid activeValidatorCount: ${activeValidatorCount} > ${activeIndices.length}`);
+  }
+  // only the first `activeValidatorCount` elements are copied to `activeIndices`
+  const _activeIndices = new Uint32Array(activeValidatorCount);
+  for (let i = 0; i < activeValidatorCount; i++) {
+    _activeIndices[i] = activeIndices[i];
+  }
   const shuffling = _activeIndices.slice();
   unshuffleList(shuffling, seed);
 
-  const activeValidatorCount = activeIndices.length;
   const committeesPerSlot = computeCommitteeCount(activeValidatorCount);
 
   const committeeCount = committeesPerSlot * SLOTS_PER_EPOCH;

packages/state-transition/test/perf/epoch/utilPhase0.ts

@@ -1,4 +1,4 @@
-import {AttesterFlags, FLAG_ELIGIBLE_ATTESTER, hasMarkers, toAttesterFlags} from "../../../src/index.js";
+import {AttesterFlags, toAttesterFlags} from "../../../src/index.js";
 import {CachedBeaconStatePhase0, CachedBeaconStateAltair, EpochTransitionCache} from "../../../src/types.js";
 
 /**
@@ -14,18 +14,11 @@ export function generateBalanceDeltasEpochTransitionCache(
   const vc = state.validators.length;
 
   const {proposerIndices, inclusionDelays, flags} = generateStatuses(state.validators.length, flagFactors);
-  const eligibleValidatorIndices: number[] = [];
-  for (let i = 0; i < flags.length; i++) {
-    if (hasMarkers(flags[i], FLAG_ELIGIBLE_ATTESTER)) {
-      eligibleValidatorIndices.push(i);
-    }
-  }
 
   const cache: Partial<EpochTransitionCache> = {
     proposerIndices,
     inclusionDelays,
     flags,
-    eligibleValidatorIndices,
     totalActiveStakeByIncrement: vc,
     baseRewardPerIncrement: 726,
     prevEpochUnslashedStake: {

packages/state-transition/test/perf/util/shufflings.test.ts

@@ -35,7 +35,8 @@ describe("epoch shufflings", () => {
   itBench({
     id: `computeEpochShuffling - vc ${numValidators}`,
     fn: () => {
-      computeEpochShuffling(state, state.epochCtx.nextShuffling.activeIndices, nextEpoch);
+      const {activeIndices} = state.epochCtx.nextShuffling;
+      computeEpochShuffling(state, activeIndices, activeIndices.length, nextEpoch);
     },
   });