From ac534d85359dd2dc35cbd182d34c98e431b7a17d Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 27 Aug 2024 13:01:34 -0300 Subject: [PATCH 1/5] feat: Add CLI command for gathering proving metrics --- yarn-project/archiver/src/archiver/index.ts | 1 + yarn-project/cli/package.json | 5 +- yarn-project/cli/src/cmds/l1/index.ts | 23 ++++++++- yarn-project/cli/src/cmds/l1/prover_stats.ts | 50 ++++++++++++++++++++ yarn-project/yarn.lock | 2 + 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 yarn-project/cli/src/cmds/l1/prover_stats.ts diff --git a/yarn-project/archiver/src/archiver/index.ts b/yarn-project/archiver/src/archiver/index.ts index 81aa8727e17..45ca819e882 100644 --- a/yarn-project/archiver/src/archiver/index.ts +++ b/yarn-project/archiver/src/archiver/index.ts @@ -4,3 +4,4 @@ export { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_sto export { ArchiverDataStore } from './archiver_store.js'; export { KVArchiverDataStore } from './kv_archiver_store/kv_archiver_store.js'; export { ContractInstanceStore } from './kv_archiver_store/contract_instance_store.js'; +export { retrieveL2ProofVerifiedEvents } from './structs/data_retrieval.js'; diff --git a/yarn-project/cli/package.json b/yarn-project/cli/package.json index 4a6033e3a5f..08b2a4a4b9a 100644 --- a/yarn-project/cli/package.json +++ b/yarn-project/cli/package.json @@ -63,6 +63,7 @@ ] }, "dependencies": { + "@aztec/archiver": "workspace:^", "@aztec/aztec.js": "workspace:^", "@aztec/circuit-types": "workspace:^", "@aztec/foundation": "workspace:^", @@ -71,6 +72,7 @@ "@iarna/toml": "^2.2.5", "@libp2p/peer-id-factory": "^3.0.4", "commander": "^12.1.0", + "lodash.groupby": "^4.6.0", "semver": "^7.5.4", "solc": "^0.8.26", "source-map-support": "^0.5.21", @@ -84,6 +86,7 @@ "@aztec/protocol-contracts": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", + "@types/lodash.groupby": "^4.6.9", "@types/lodash.startcase": "^4.4.7", "@types/node": "^18.7.23", "@types/semver": "^7.5.2", @@ -113,4 +116,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/yarn-project/cli/src/cmds/l1/index.ts b/yarn-project/cli/src/cmds/l1/index.ts index 4ebfeb587ba..f3ca73f3683 100644 --- a/yarn-project/cli/src/cmds/l1/index.ts +++ b/yarn-project/cli/src/cmds/l1/index.ts @@ -159,7 +159,7 @@ export function injectCommands(program: Command, log: LogFn, debugLogger: DebugL }); program - .command('set-proven-until') + .command('set-proven-until', { hidden: true }) .description( 'Instructs the L1 rollup contract to assume all blocks until the given number are automatically proven.', ) @@ -190,5 +190,26 @@ export function injectCommands(program: Command, log: LogFn, debugLogger: DebugL ); }); + program + .command('prover-stats', { hidden: true }) + .requiredOption( + '--l1-rpc-url ', + 'Url of the ethereum host. Chain identifiers localhost and testnet can be used', + ETHEREUM_HOST, + ) + .addOption(l1ChainIdOption) + .option('--start-block ', 'The block number to start from', parseBigint, 1n) + .option('--batch-size ', 'The number of blocks to query in each batch', parseBigint, 100n) + .option('--l1-rollup-address ', 'Address of the rollup contract (required if node URL is not set)') + .option( + '--node-url ', + 'JSON RPC URL of an Aztec node to retrieve the rollup contract address (required if L1 rollup address is not set)', + ) + .action(async options => { + const { proverStats } = await import('./prover_stats.js'); + const { l1RpcUrl, chainId, l1RollupAddress, startBlock, batchSize, nodeUrl } = options; + await proverStats({ l1RpcUrl, chainId, l1RollupAddress, startBlock, batchSize, nodeUrl, log }); + }); + return program; } diff --git a/yarn-project/cli/src/cmds/l1/prover_stats.ts b/yarn-project/cli/src/cmds/l1/prover_stats.ts new file mode 100644 index 00000000000..4cda8a13a45 --- /dev/null +++ b/yarn-project/cli/src/cmds/l1/prover_stats.ts @@ -0,0 +1,50 @@ +import { retrieveL2ProofVerifiedEvents } from '@aztec/archiver'; +import { createAztecNodeClient } from '@aztec/circuit-types'; +import { EthAddress } from '@aztec/circuits.js'; +import { createEthereumChain } from '@aztec/ethereum'; +import { type LogFn, createDebugLogger } from '@aztec/foundation/log'; + +import groupBy from 'lodash.groupby'; +import { createPublicClient, http } from 'viem'; + +export async function proverStats(opts: { + l1RpcUrl: string; + chainId: number; + l1RollupAddress: string | undefined; + nodeUrl: string | undefined; + log: LogFn; + startBlock: bigint; + batchSize: bigint; +}) { + const debugLog = createDebugLogger('aztec:cli:prover_stats'); + const { startBlock, chainId, l1RpcUrl, l1RollupAddress, batchSize, nodeUrl, log } = opts; + if (!l1RollupAddress && !nodeUrl) { + throw new Error('Either L1 rollup address or node URL must be set'); + } + const rollup = l1RollupAddress + ? EthAddress.fromString(l1RollupAddress) + : await createAztecNodeClient(nodeUrl!) + .getL1ContractAddresses() + .then(a => a.rollupAddress); + + const chain = createEthereumChain(l1RpcUrl, chainId).chainInfo; + const publicClient = createPublicClient({ chain, transport: http(l1RpcUrl) }); + const lastBlockNum = await publicClient.getBlockNumber(); + debugLog.verbose(`Querying events on rollup at ${rollup.toString()} from ${startBlock} up to ${lastBlockNum}`); + + let blockNum = startBlock; + const events = []; + while (blockNum <= lastBlockNum) { + const end = blockNum + batchSize > lastBlockNum + 1n ? lastBlockNum + 1n : blockNum + batchSize; + debugLog.verbose(`Querying events from block ${blockNum} to ${end}`); + const newEvents = await retrieveL2ProofVerifiedEvents(publicClient, rollup, blockNum, end); + events.push(...newEvents); + debugLog.verbose(`Got ${newEvents.length} events`); + blockNum += batchSize; + } + + const stats = groupBy(events, 'proverId'); + for (const proverId in stats) { + log(`${proverId}, ${stats[proverId].length}`); + } +} diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index e325bdd7b59..78782943b75 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -460,6 +460,7 @@ __metadata: "@jest/globals": ^29.5.0 "@libp2p/peer-id-factory": ^3.0.4 "@types/jest": ^29.5.0 + "@types/lodash.groupby": ^4.6.9 "@types/lodash.startcase": ^4.4.7 "@types/node": ^18.7.23 "@types/semver": ^7.5.2 @@ -467,6 +468,7 @@ __metadata: commander: ^12.1.0 jest: ^29.5.0 jest-mock-extended: ^3.0.5 + lodash.groupby: ^4.6.0 semver: ^7.5.4 solc: ^0.8.26 source-map-support: ^0.5.21 From f2701fcd9a3420142381829303e338d7a8c0f35a Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 27 Aug 2024 13:45:59 -0300 Subject: [PATCH 2/5] Fix build --- yarn-project/cli/package.json | 2 +- yarn-project/cli/tsconfig.json | 9 ++++++--- yarn-project/yarn.lock | 17 +++++++++-------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/yarn-project/cli/package.json b/yarn-project/cli/package.json index 08b2a4a4b9a..7c9959d2b1e 100644 --- a/yarn-project/cli/package.json +++ b/yarn-project/cli/package.json @@ -116,4 +116,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/cli/tsconfig.json b/yarn-project/cli/tsconfig.json index 69a3c280d81..4d6ad7125a3 100644 --- a/yarn-project/cli/tsconfig.json +++ b/yarn-project/cli/tsconfig.json @@ -6,6 +6,9 @@ "tsBuildInfoFile": ".tsbuildinfo" }, "references": [ + { + "path": "../archiver" + }, { "path": "../aztec.js" }, @@ -15,6 +18,9 @@ { "path": "../foundation" }, + { + "path": "../l1-artifacts" + }, { "path": "../types" }, @@ -27,9 +33,6 @@ { "path": "../ethereum" }, - { - "path": "../l1-artifacts" - }, { "path": "../protocol-contracts" } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 78782943b75..533c90985fa 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -448,6 +448,7 @@ __metadata: resolution: "@aztec/cli@workspace:cli" dependencies: "@aztec/accounts": "workspace:^" + "@aztec/archiver": "workspace:^" "@aztec/aztec.js": "workspace:^" "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" @@ -3329,7 +3330,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noir_codegen@portal:../noir/packages/noir_codegen::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.33.0 + "@noir-lang/types": 0.32.0 glob: ^10.3.10 ts-command-line-args: ^2.5.1 bin: @@ -3338,13 +3339,13 @@ __metadata: linkType: soft "@noir-lang/noir_js@file:../noir/packages/noir_js::locator=%40aztec%2Faztec3-packages%40workspace%3A.": - version: 0.33.0 - resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=e4c9c2&locator=%40aztec%2Faztec3-packages%40workspace%3A." + version: 0.32.0 + resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=bd4828&locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/acvm_js": 0.49.0 - "@noir-lang/noirc_abi": 0.33.0 - "@noir-lang/types": 0.33.0 - checksum: 82d759db2487b4c2fd298ec989cb0f4547f5dfef4cc70bf37531e1d92e04bfecf425576497b8f63769a3249a584a1a31c2751615a2633a51b2dd815e296044b1 + "@noir-lang/acvm_js": 0.48.0 + "@noir-lang/noirc_abi": 0.32.0 + "@noir-lang/types": 0.32.0 + checksum: fd5f51c35854f793f8ec20f8437a89a4cf87562f80adfd2870a56535f5faef4c75ce9a3f2f3daaa6ab3a77d125c18b55bd35247ff33d97c1d3f8d76f5b440749 languageName: node linkType: hard @@ -3352,7 +3353,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noirc_abi@portal:../noir/packages/noirc_abi::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.33.0 + "@noir-lang/types": 0.32.0 languageName: node linkType: soft From e97f91f0009ba511ce3750cfcb32d3e3c73df952 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 27 Aug 2024 14:54:55 -0300 Subject: [PATCH 3/5] Fix build again --- yarn-project/yarn.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 533c90985fa..55e67733f99 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -3330,7 +3330,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noir_codegen@portal:../noir/packages/noir_codegen::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.32.0 + "@noir-lang/types": 0.33.0 glob: ^10.3.10 ts-command-line-args: ^2.5.1 bin: @@ -3339,13 +3339,13 @@ __metadata: linkType: soft "@noir-lang/noir_js@file:../noir/packages/noir_js::locator=%40aztec%2Faztec3-packages%40workspace%3A.": - version: 0.32.0 - resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=bd4828&locator=%40aztec%2Faztec3-packages%40workspace%3A." + version: 0.33.0 + resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=e4c9c2&locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/acvm_js": 0.48.0 - "@noir-lang/noirc_abi": 0.32.0 - "@noir-lang/types": 0.32.0 - checksum: fd5f51c35854f793f8ec20f8437a89a4cf87562f80adfd2870a56535f5faef4c75ce9a3f2f3daaa6ab3a77d125c18b55bd35247ff33d97c1d3f8d76f5b440749 + "@noir-lang/acvm_js": 0.49.0 + "@noir-lang/noirc_abi": 0.33.0 + "@noir-lang/types": 0.33.0 + checksum: 82d759db2487b4c2fd298ec989cb0f4547f5dfef4cc70bf37531e1d92e04bfecf425576497b8f63769a3249a584a1a31c2751615a2633a51b2dd815e296044b1 languageName: node linkType: hard @@ -3353,7 +3353,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noirc_abi@portal:../noir/packages/noirc_abi::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.32.0 + "@noir-lang/types": 0.33.0 languageName: node linkType: soft From 80f89822f562cb8acec5369f0b003656435d5f67 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 27 Aug 2024 15:12:05 -0300 Subject: [PATCH 4/5] Fix build yet again --- yarn-project/archiver/src/archiver/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/index.ts b/yarn-project/archiver/src/archiver/index.ts index 45ca819e882..7b6ae99fb56 100644 --- a/yarn-project/archiver/src/archiver/index.ts +++ b/yarn-project/archiver/src/archiver/index.ts @@ -4,4 +4,4 @@ export { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_sto export { ArchiverDataStore } from './archiver_store.js'; export { KVArchiverDataStore } from './kv_archiver_store/kv_archiver_store.js'; export { ContractInstanceStore } from './kv_archiver_store/contract_instance_store.js'; -export { retrieveL2ProofVerifiedEvents } from './structs/data_retrieval.js'; +export { retrieveL2ProofVerifiedEvents } from './data_retrieval.js'; From ce5a17a13c44c9676e69f133a94273a132350584 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 27 Aug 2024 15:51:27 -0300 Subject: [PATCH 5/5] Fix build yet another time --- yarn-project/archiver/src/archiver/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn-project/archiver/src/archiver/index.ts b/yarn-project/archiver/src/archiver/index.ts index 7b6ae99fb56..81aa8727e17 100644 --- a/yarn-project/archiver/src/archiver/index.ts +++ b/yarn-project/archiver/src/archiver/index.ts @@ -4,4 +4,3 @@ export { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_sto export { ArchiverDataStore } from './archiver_store.js'; export { KVArchiverDataStore } from './kv_archiver_store/kv_archiver_store.js'; export { ContractInstanceStore } from './kv_archiver_store/contract_instance_store.js'; -export { retrieveL2ProofVerifiedEvents } from './data_retrieval.js';