From 36be6b39d1c290dd2884887e37313a11792de19b Mon Sep 17 00:00:00 2001 From: Phil Ngo Date: Thu, 28 Nov 2024 12:54:20 -0500 Subject: [PATCH 1/6] chore: bump package versions to 1.23.1 --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 8 ++++---- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 +++++++------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 12 ++++++------ packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 6 +++--- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 100 insertions(+), 100 deletions(-) diff --git a/lerna.json b/lerna.json index 6d76d7f4a488..cdd74f75b2cf 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.23.0", + "version": "1.23.1", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index a787147f6f07..1f4f3e7c3fdf 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -72,10 +72,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.18.0", - "@lodestar/config": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/config": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 87d80d27039c..ffce2f3eacfe 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -120,18 +120,18 @@ "@libp2p/peer-id-factory": "^4.1.0", "@libp2p/prometheus-metrics": "^3.0.21", "@libp2p/tcp": "9.0.23", - "@lodestar/api": "^1.23.0", - "@lodestar/config": "^1.23.0", - "@lodestar/db": "^1.23.0", - "@lodestar/fork-choice": "^1.23.0", - "@lodestar/light-client": "^1.23.0", - "@lodestar/logger": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/reqresp": "^1.23.0", - "@lodestar/state-transition": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", - "@lodestar/validator": "^1.23.0", + "@lodestar/api": "^1.23.1", + "@lodestar/config": "^1.23.1", + "@lodestar/db": "^1.23.1", + "@lodestar/fork-choice": "^1.23.1", + "@lodestar/light-client": "^1.23.1", + "@lodestar/logger": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/reqresp": "^1.23.1", + "@lodestar/state-transition": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", + "@lodestar/validator": "^1.23.1", "@multiformats/multiaddr": "^12.1.3", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 287300183a7f..e1b2134f5f82 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.23.0", + "version": "1.23.1", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -62,17 +62,17 @@ "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", "@libp2p/peer-id-factory": "^4.1.0", - "@lodestar/api": "^1.23.0", - "@lodestar/beacon-node": "^1.23.0", - "@lodestar/config": "^1.23.0", - "@lodestar/db": "^1.23.0", - "@lodestar/light-client": "^1.23.0", - "@lodestar/logger": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/state-transition": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", - "@lodestar/validator": "^1.23.0", + "@lodestar/api": "^1.23.1", + "@lodestar/beacon-node": "^1.23.1", + "@lodestar/config": "^1.23.1", + "@lodestar/db": "^1.23.1", + "@lodestar/light-client": "^1.23.1", + "@lodestar/logger": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/state-transition": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", + "@lodestar/validator": "^1.23.1", "@multiformats/multiaddr": "^12.1.3", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -88,7 +88,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.23.0", + "@lodestar/test-utils": "^1.23.1", "@types/debug": "^4.1.7", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", diff --git a/packages/config/package.json b/packages/config/package.json index aadbf9e004b2..5a4edcf7879c 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.23.0", + "version": "1.23.1", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,8 +65,8 @@ ], "dependencies": { "@chainsafe/ssz": "^0.18.0", - "@lodestar/params": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0" + "@lodestar/params": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1" } } diff --git a/packages/db/package.json b/packages/db/package.json index 370caaa5f8c9..a332d316185b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.23.0", + "version": "1.23.1", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -36,12 +36,12 @@ }, "dependencies": { "@chainsafe/ssz": "^0.18.0", - "@lodestar/config": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/config": "^1.23.1", + "@lodestar/utils": "^1.23.1", "classic-level": "^1.4.1", "it-all": "^3.0.4" }, "devDependencies": { - "@lodestar/logger": "^1.23.0" + "@lodestar/logger": "^1.23.1" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index 1ec88f28ac4d..c3d262173321 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.23.0", + "version": "1.23.1", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/blst": "^2.1.0", - "@lodestar/api": "^1.23.0", - "@lodestar/config": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/state-transition": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/api": "^1.23.1", + "@lodestar/config": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/state-transition": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 1ee9d043c925..cc86cd75b45f 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -37,11 +37,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.18.0", - "@lodestar/config": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/state-transition": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0" + "@lodestar/config": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/state-transition": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index c942ff07ed44..f972250937ae 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -77,11 +77,11 @@ "@chainsafe/blst": "^0.2.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.18.0", - "@lodestar/api": "^1.23.0", - "@lodestar/config": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/api": "^1.23.1", + "@lodestar/config": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", "mitt": "^3.0.0" }, "devDependencies": { diff --git a/packages/logger/package.json b/packages/logger/package.json index b9e9beef39be..3e8d2095866b 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.23.0", + "@lodestar/utils": "^1.23.1", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.23.0", + "@lodestar/test-utils": "^1.23.1", "@types/triple-beam": "^1.3.2", "triple-beam": "^1.3.0" }, diff --git a/packages/params/package.json b/packages/params/package.json index ce78e826ef18..203dd660571e 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.23.0", + "version": "1.23.1", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index eee42f4226c2..cbaa7dabdbf3 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.23.0", - "@lodestar/config": "^1.23.0", - "@lodestar/light-client": "^1.23.0", - "@lodestar/logger": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/api": "^1.23.1", + "@lodestar/config": "^1.23.1", + "@lodestar/light-client": "^1.23.1", + "@lodestar/logger": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", "ethereum-cryptography": "^2.0.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.23.0", + "@lodestar/test-utils": "^1.23.1", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 981016a363df..c2280ee8fc37 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -54,9 +54,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.3.0", - "@lodestar/config": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/config": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/utils": "^1.23.1", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -65,8 +65,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.23.0", - "@lodestar/types": "^1.23.0", + "@lodestar/logger": "^1.23.1", + "@lodestar/types": "^1.23.1", "libp2p": "1.4.3" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 6f58e55d2201..c35814d8b57f 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.23.0", + "version": "1.23.1", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,7 +62,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.23.0", + "@lodestar/utils": "^1.23.1", "axios": "^1.3.4", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 5dfecfa9bbf6..31d638bb0b7e 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -65,10 +65,10 @@ "@chainsafe/pubkey-index-map": "2.0.0", "@chainsafe/ssz": "^0.18.0", "@chainsafe/swap-or-not-shuffle": "^0.0.2", - "@lodestar/config": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/config": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", "bigint-buffer": "^1.1.5", "immutable": "^4.3.2" }, diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index c8fc5cd41c55..ff65198c3b35 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.23.0", + "version": "1.23.1", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -59,8 +59,8 @@ "dependencies": { "@chainsafe/bls-keystore": "^3.1.0", "@chainsafe/blst": "^2.1.0", - "@lodestar/params": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/params": "^1.23.1", + "@lodestar/utils": "^1.23.1", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index 9201ef2f4e88..fbaf060afbca 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": { ".": { @@ -74,7 +74,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.18.0", - "@lodestar/params": "^1.23.0", + "@lodestar/params": "^1.23.1", "ethereum-cryptography": "^2.0.0" }, "keywords": [ diff --git a/packages/utils/package.json b/packages/utils/package.json index 9b6f5e797e68..babd09066e8c 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.23.0", + "version": "1.23.1", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index 1566c08668b9..823d2dd86907 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.23.0", + "version": "1.23.1", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -47,17 +47,17 @@ "dependencies": { "@chainsafe/blst": "^2.1.0", "@chainsafe/ssz": "^0.18.0", - "@lodestar/api": "^1.23.0", - "@lodestar/config": "^1.23.0", - "@lodestar/db": "^1.23.0", - "@lodestar/params": "^1.23.0", - "@lodestar/state-transition": "^1.23.0", - "@lodestar/types": "^1.23.0", - "@lodestar/utils": "^1.23.0", + "@lodestar/api": "^1.23.1", + "@lodestar/config": "^1.23.1", + "@lodestar/db": "^1.23.1", + "@lodestar/params": "^1.23.1", + "@lodestar/state-transition": "^1.23.1", + "@lodestar/types": "^1.23.1", + "@lodestar/utils": "^1.23.1", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.23.0", + "@lodestar/test-utils": "^1.23.1", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From ffa4bb8cc1f34ac82ef0aa1248a359cdff47c7b8 Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 27 Nov 2024 15:22:24 +0700 Subject: [PATCH 2/6] fix: prune checkpoint states at syncing time (#7241) * fix: prune checkpoint states at syncing time * fix: lint * fix: check-types in test --- packages/beacon-node/src/chain/chain.ts | 1 - .../stateCache/persistentCheckpointsCache.ts | 27 +++---------------- .../persistentCheckpointsCache.test.ts | 18 +++++-------- 3 files changed, 9 insertions(+), 37 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 144b73f0c01d..9141aa994224 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -293,7 +293,6 @@ export class BeaconChain implements IBeaconChain { metrics, logger, clock, - shufflingCache: this.shufflingCache, blockStateCache, bufferPool: this.bufferPool, datastore: fileDataStore diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 0719efcfd309..8663c0533ed8 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -6,7 +6,6 @@ import {loadCachedBeaconState} from "@lodestar/state-transition"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; import {Metrics} from "../../metrics/index.js"; import {IClock} from "../../util/clock.js"; -import {ShufflingCache} from "../shufflingCache.js"; import {AllocSource, BufferPool, BufferWithKey} from "../../util/bufferPool.js"; import {StateCloneOpts} from "../regen/interface.js"; import {serializeState} from "../serializeState.js"; @@ -17,8 +16,6 @@ import {CheckpointHex, CacheItemType, CheckpointStateCache, BlockStateCache} fro export type PersistentCheckpointStateCacheOpts = { /** Keep max n states in memory, persist the rest to disk */ maxCPStateEpochsInMemory?: number; - /** for testing only */ - processLateBlock?: boolean; }; type PersistentCheckpointStateCacheModules = { @@ -26,7 +23,6 @@ type PersistentCheckpointStateCacheModules = { logger: Logger; clock?: IClock | null; signal?: AbortSignal; - shufflingCache: ShufflingCache; datastore: CPStateDatastore; blockStateCache: BlockStateCache; bufferPool?: BufferPool | null; @@ -102,24 +98,12 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { private preComputedCheckpoint: string | null = null; private preComputedCheckpointHits: number | null = null; private readonly maxEpochsInMemory: number; - // only for testing, default false for production - private readonly processLateBlock: boolean; private readonly datastore: CPStateDatastore; - private readonly shufflingCache: ShufflingCache; private readonly blockStateCache: BlockStateCache; private readonly bufferPool?: BufferPool | null; constructor( - { - metrics, - logger, - clock, - signal, - shufflingCache, - datastore, - blockStateCache, - bufferPool, - }: PersistentCheckpointStateCacheModules, + {metrics, logger, clock, signal, datastore, blockStateCache, bufferPool}: PersistentCheckpointStateCacheModules, opts: PersistentCheckpointStateCacheOpts ) { this.cache = new MapTracker(metrics?.cpStateCache); @@ -153,10 +137,8 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { throw new Error("maxEpochsInMemory must be >= 0"); } this.maxEpochsInMemory = opts.maxCPStateEpochsInMemory ?? DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY; - this.processLateBlock = opts.processLateBlock ?? false; // Specify different datastore for testing this.datastore = datastore; - this.shufflingCache = shufflingCache; this.blockStateCache = blockStateCache; this.bufferPool = bufferPool; } @@ -487,12 +469,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // 2/3 of slot is the most free time of every slot, take that chance to persist checkpoint states // normally it should only persist checkpoint states at 2/3 of slot 0 of epoch await sleep(secToTwoThirdsSlot * 1000, this.signal); - } else if (!this.processLateBlock) { - // normally the block persist happens at 2/3 of slot 0 of epoch, if it's already late then just skip to allow other tasks to run - // there are plenty of chances in the same epoch to persist checkpoint states, also if block is late it could be reorged - this.logger.verbose("Skip persist checkpoint states", {blockSlot, root: blockRootHex}); - return 0; } + // at syncing time, it's critical to persist checkpoint states as soon as possible to avoid OOM during unfinality time + // if node is synced this is not a hot time because block comes late, we'll likely miss attestation already, or the block is orphaned const persistEpochs = sortedEpochs.slice(0, sortedEpochs.length - this.maxEpochsInMemory); for (const lowestEpoch of persistEpochs) { diff --git a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts index f98b180fa983..aecec564bd92 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts @@ -90,10 +90,9 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 2, processLateBlock: true} + {maxCPStateEpochsInMemory: 2} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -165,10 +164,9 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 2, processLateBlock: true} + {maxCPStateEpochsInMemory: 2} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -242,10 +240,9 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 2, processLateBlock: true} + {maxCPStateEpochsInMemory: 2} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -548,10 +545,9 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 1, processLateBlock: true} + {maxCPStateEpochsInMemory: 1} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -820,10 +816,9 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 0, processLateBlock: true} + {maxCPStateEpochsInMemory: 0} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -911,10 +906,9 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 0, processLateBlock: true} + {maxCPStateEpochsInMemory: 0} ); const root1a = Buffer.alloc(32, 100); From 2c2326f2debd6b181669967aff17ead5e5c02826 Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 27 Nov 2024 15:52:43 +0700 Subject: [PATCH 3/6] fix: sync cached isCompoundingValidatorArr at epoch transition (#7247) --- .../src/cache/epochTransitionCache.ts | 47 ++++++++++++------- .../epoch/processEffectiveBalanceUpdates.ts | 7 +-- .../src/epoch/processPendingDeposits.ts | 5 +- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 2a35317fbc93..b21f940c28d5 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,27 +1,32 @@ -import {phase0, Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; -import {intDiv, toRootHex} from "@lodestar/utils"; import { EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, - SLOTS_PER_HISTORICAL_ROOT, MIN_ACTIVATION_BALANCE, + SLOTS_PER_HISTORICAL_ROOT, } from "@lodestar/params"; +import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; +import {intDiv, toRootHex} from "@lodestar/utils"; +import {processPendingAttestations} from "../epoch/processPendingAttestations.js"; import { - hasMarkers, - FLAG_UNSLASHED, + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + CachedBeaconStatePhase0, + hasCompoundingWithdrawalCredential, +} from "../index.js"; +import {computeBaseRewardPerIncrement} from "../util/altair.js"; +import { + FLAG_CURR_HEAD_ATTESTER, + FLAG_CURR_SOURCE_ATTESTER, + FLAG_CURR_TARGET_ATTESTER, FLAG_ELIGIBLE_ATTESTER, + FLAG_PREV_HEAD_ATTESTER, FLAG_PREV_SOURCE_ATTESTER, FLAG_PREV_TARGET_ATTESTER, - FLAG_PREV_HEAD_ATTESTER, - FLAG_CURR_SOURCE_ATTESTER, - FLAG_CURR_TARGET_ATTESTER, - FLAG_CURR_HEAD_ATTESTER, + FLAG_UNSLASHED, + hasMarkers, } from "../util/attesterStatus.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../index.js"; -import {computeBaseRewardPerIncrement} from "../util/altair.js"; -import {processPendingAttestations} from "../epoch/processPendingAttestations.js"; export type EpochTransitionCacheOpts = { /** @@ -133,11 +138,7 @@ export interface EpochTransitionCache { flags: number[]; - /** - * Validators in the current epoch, should use it for read-only value instead of accessing state.validators directly. - * Note that during epoch processing, validators could be updated so need to use it with care. - */ - validators: phase0.Validator[]; + isCompoundingValidatorArr: boolean[]; /** * balances array will be populated by processRewardsAndPenalties() and consumed by processEffectiveBalanceUpdates(). @@ -209,6 +210,8 @@ const inclusionDelays = new Array(); /** WARNING: reused, never gc'd */ const flags = new Array(); /** WARNING: reused, never gc'd */ +const isCompoundingValidatorArr = new Array(); +/** WARNING: reused, never gc'd */ const nextEpochShufflingActiveValidatorIndices = new Array(); export function beforeProcessEpoch( @@ -262,6 +265,10 @@ export function beforeProcessEpoch( // TODO: optimize by combining the two loops // likely will require splitting into phase0 and post-phase0 versions + if (forkSeq >= ForkSeq.electra) { + isCompoundingValidatorArr.length = validatorCount; + } + // Clone before being mutated in processEffectiveBalanceUpdates epochCtx.beforeEpochTransition(); @@ -298,6 +305,10 @@ export function beforeProcessEpoch( flags[i] = flag; + if (forkSeq >= ForkSeq.electra) { + isCompoundingValidatorArr[i] = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials); + } + if (isActiveCurr) { totalActiveStakeByIncrement += effectiveBalancesByIncrements[i]; } else { @@ -511,7 +522,7 @@ export function beforeProcessEpoch( proposerIndices, inclusionDelays, flags, - validators, + isCompoundingValidatorArr, // Will be assigned in processRewardsAndPenalties() balances: undefined, }; diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index 1fe5c92eea1a..6e71d8ccdc73 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -5,10 +5,11 @@ import { HYSTERESIS_QUOTIENT, HYSTERESIS_UPWARD_MULTIPLIER, MAX_EFFECTIVE_BALANCE, + MAX_EFFECTIVE_BALANCE_ELECTRA, + MIN_ACTIVATION_BALANCE, TIMELY_TARGET_FLAG_INDEX, } from "@lodestar/params"; import {EpochTransitionCache, CachedBeaconStateAllForks, BeaconStateAltair} from "../types.js"; -import {getMaxEffectiveBalance} from "../util/validator.js"; /** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; @@ -43,7 +44,7 @@ export function processEffectiveBalanceUpdates( // and updated in processPendingDeposits() and processPendingConsolidations() // so it's recycled here for performance. const balances = cache.balances ?? state.balances.getAll(); - const currentEpochValidators = cache.validators; + const isCompoundingValidatorArr = cache.isCompoundingValidatorArr; let numUpdate = 0; for (let i = 0, len = balances.length; i < len; i++) { @@ -58,7 +59,7 @@ export function processEffectiveBalanceUpdates( effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; } else { // from electra, effectiveBalanceLimit is per validator - effectiveBalanceLimit = getMaxEffectiveBalance(currentEpochValidators[i].withdrawalCredentials); + effectiveBalanceLimit = isCompoundingValidatorArr[i] ? MAX_EFFECTIVE_BALANCE_ELECTRA : MIN_ACTIVATION_BALANCE; } if ( diff --git a/packages/state-transition/src/epoch/processPendingDeposits.ts b/packages/state-transition/src/epoch/processPendingDeposits.ts index 53af3ab38763..b25cf1d040a4 100644 --- a/packages/state-transition/src/epoch/processPendingDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingDeposits.ts @@ -2,8 +2,9 @@ import {FAR_FUTURE_EPOCH, ForkSeq, GENESIS_SLOT, MAX_PENDING_DEPOSITS_PER_EPOCH} import {PendingDeposit} from "@lodestar/types/lib/electra/types.js"; import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js"; import {increaseBalance} from "../util/balance.js"; -import {getActivationExitChurnLimit} from "../util/validator.js"; +import {hasCompoundingWithdrawalCredential} from "../util/electra.js"; import {computeStartSlotAtEpoch} from "../util/epoch.js"; +import {getActivationExitChurnLimit} from "../util/validator.js"; import {addValidatorToRegistry, isValidDepositSignature} from "../block/processDeposit.js"; /** @@ -106,6 +107,8 @@ function applyPendingDeposit( // Verify the deposit signature (proof of possession) which is not checked by the deposit contract if (isValidDepositSignature(state.config, pubkey, withdrawalCredentials, amount, signature)) { addValidatorToRegistry(ForkSeq.electra, state, pubkey, withdrawalCredentials, amount); + const newValidatorIndex = state.validators.length - 1; + cache.isCompoundingValidatorArr[newValidatorIndex] = hasCompoundingWithdrawalCredential(withdrawalCredentials); } } else { // Increase balance From f1e81982a3df27a88d107137e79556225db46015 Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 27 Nov 2024 17:01:15 +0700 Subject: [PATCH 4/6] fix: handle outOfRangeData when range sync Deneb (#7249) * fix: handle outOfRangeData for beaconBlocksMaybeBlobsByRange() * fix: lint * fix: archiveBlocks - handle deneb outOfRangeData block --- .../src/chain/archiver/archiveBlocks.ts | 44 ++++++++++++++----- .../reqresp/beaconBlocksMaybeBlobsByRange.ts | 8 ++-- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts index 8e1ac456579a..320715f09dda 100644 --- a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts @@ -59,8 +59,8 @@ export async function archiveBlocks( }); if (finalizedPostDeneb) { - await migrateBlobSidecarsFromHotToColdDb(config, db, finalizedCanonicalBlockRoots); - logger.verbose("Migrated blobSidecars from hot DB to cold DB"); + const migrate = await migrateBlobSidecarsFromHotToColdDb(config, db, finalizedCanonicalBlockRoots, currentEpoch); + logger.verbose(migrate ? "Migrated blobSidecars from hot DB to cold DB" : "Skip blobSidecars migration"); } } @@ -157,22 +157,36 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot } } +/** + * Migrate blobSidecars from hot db to cold db. + * @returns true if we do that, false if block is out of range data. + */ async function migrateBlobSidecarsFromHotToColdDb( config: ChainForkConfig, db: IBeaconDb, - blocks: BlockRootSlot[] -): Promise { + blocks: BlockRootSlot[], + currentEpoch: Epoch +): Promise { + let result = false; + for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) { const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length); const canonicalBlocks = blocks.slice(i, toIdx); // processCanonicalBlocks - if (canonicalBlocks.length === 0) return; + if (canonicalBlocks.length === 0) return false; // load Buffer instead of ssz deserialized to improve performance const canonicalBlobSidecarsEntries: KeyValue[] = await Promise.all( canonicalBlocks - .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.deneb) + .filter((block) => { + const blockSlot = block.slot; + const blockEpoch = computeEpochAtSlot(blockSlot); + return ( + config.getForkSeq(blockSlot) >= ForkSeq.deneb && + blockEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS + ); + }) .map(async (block) => { const bytes = await db.blobSidecars.getBinary(block.root); if (!bytes) { @@ -182,12 +196,20 @@ async function migrateBlobSidecarsFromHotToColdDb( }) ); - // put to blockArchive db and delete block db - await Promise.all([ - db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries), - db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)), - ]); + const migrate = canonicalBlobSidecarsEntries.length > 0; + + if (migrate) { + // put to blockArchive db and delete block db + await Promise.all([ + db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries), + db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)), + ]); + } + + result = result || migrate; } + + return result; } /** diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts index 3d71087c8fd8..041f1c59cc0a 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts @@ -36,8 +36,9 @@ export async function beaconBlocksMaybeBlobsByRange( return blocks.map((block) => getBlockInput.preData(config, block.data, BlockSource.byRange, block.bytes)); } + // From Deneb // Only request blobs if they are recent enough - if (computeEpochAtSlot(startSlot) >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS) { + if (startEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS) { const [allBlocks, allBlobSidecars] = await Promise.all([ network.sendBeaconBlocksByRange(peerId, request), network.sendBlobSidecarsByRange(peerId, request), @@ -46,8 +47,9 @@ export async function beaconBlocksMaybeBlobsByRange( return matchBlockWithBlobs(config, allBlocks, allBlobSidecars, endSlot, BlockSource.byRange, BlobsSource.byRange); } - // Post Deneb but old blobs - throw Error("Cannot sync blobs outside of blobs prune window"); + // Data is out of range, only request blocks + const blocks = await network.sendBeaconBlocksByRange(peerId, request); + return blocks.map((block) => getBlockInput.outOfRangeData(config, block.data, BlockSource.byRange, block.bytes)); } // Assumes that the blobs are in the same sequence as blocks, doesn't require block to be sorted From 7f96e709366cbcfe860d656ca5787328debc5b2d Mon Sep 17 00:00:00 2001 From: twoeths Date: Thu, 28 Nov 2024 14:20:20 +0700 Subject: [PATCH 5/6] fix: sync cached balance when adding new validator to registry (#7255) * fix: sync cached balance when adding new validator to registry * chore: add more comments * fix: remove persisted checkpoint states from the previous run at startup --- .../chain/stateCache/persistentCheckpointsCache.ts | 11 +++++------ .../src/epoch/processPendingDeposits.ts | 6 ++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 8663c0533ed8..5573366e3523 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -151,12 +151,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { await this.datastore.init(); } const persistedKeys = await this.datastore.readKeys(); - for (const persistedKey of persistedKeys) { - const cp = datastoreKeyToCheckpoint(persistedKey); - this.cache.set(toCacheKey(cp), {type: CacheItemType.persisted, value: persistedKey}); - this.epochIndex.getOrDefault(cp.epoch).add(toRootHex(cp.root)); - } - this.logger.info("Loaded persisted checkpoint states from the last run", { + // all checkpoint states from the last run are not trusted, remove them + // otherwise if we have a bad checkpoint state from the last run, the node get stucked + // this was found during mekong devnet, see https://github.com/ChainSafe/lodestar/pull/7255 + await Promise.all(persistedKeys.map((key) => this.datastore.remove(key))); + this.logger.info("Removed persisted checkpoint states from the last run", { count: persistedKeys.length, maxEpochsInMemory: this.maxEpochsInMemory, }); diff --git a/packages/state-transition/src/epoch/processPendingDeposits.ts b/packages/state-transition/src/epoch/processPendingDeposits.ts index b25cf1d040a4..8384295350f8 100644 --- a/packages/state-transition/src/epoch/processPendingDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingDeposits.ts @@ -109,6 +109,12 @@ function applyPendingDeposit( addValidatorToRegistry(ForkSeq.electra, state, pubkey, withdrawalCredentials, amount); const newValidatorIndex = state.validators.length - 1; cache.isCompoundingValidatorArr[newValidatorIndex] = hasCompoundingWithdrawalCredential(withdrawalCredentials); + // set balance, so that the next deposit of same pubkey will increase the balance correctly + // this is to fix the double deposit issue found in mekong + // see https://github.com/ChainSafe/lodestar/pull/7255 + if (cachedBalances) { + cachedBalances[newValidatorIndex] = amount; + } } } else { // Increase balance From 9f4e7e6703866721202c68a7c740fd59786c0b66 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 27 Nov 2024 07:46:14 +0100 Subject: [PATCH 6/6] chore: skip web3_provider unit tests (#7252) --- packages/prover/test/unit/web3_provider.node.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/prover/test/unit/web3_provider.node.test.ts b/packages/prover/test/unit/web3_provider.node.test.ts index d1f281175b56..896715699e95 100644 --- a/packages/prover/test/unit/web3_provider.node.test.ts +++ b/packages/prover/test/unit/web3_provider.node.test.ts @@ -7,7 +7,8 @@ import {ProofProvider} from "../../src/proof_provider/proof_provider.js"; import {LCTransport, Web3ProviderType} from "../../src/interfaces.js"; import {JsonRpcRequest, JsonRpcRequestOrBatch, JsonRpcResponse} from "../../src/types.js"; -describe("web3_provider", () => { +// https://github.com/ChainSafe/lodestar/issues/7250 +describe.skip("web3_provider", () => { afterEach(() => { vi.clearAllMocks(); });