From 26d26437022567e2d54052f21b1c937259f26c94 Mon Sep 17 00:00:00 2001 From: esau <152162806+sklppy88@users.noreply.github.com> Date: Sat, 9 Mar 2024 18:23:47 +0100 Subject: [PATCH] feat: add api for inclusion proof of outgoing message in block #4562 (#4899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves #4562. --------- Co-authored-by: Jan Beneš --- .circleci/config.yml | 16 +++ .../aztec-node/src/aztec-node/server.ts | 47 ++++++- yarn-project/aztec.js/src/index.ts | 1 + .../aztec/src/cli/cmds/start_archiver.ts | 2 +- .../src/interfaces/aztec-node.ts | 14 +++ .../end-to-end/src/e2e_outbox.test.ts | 115 ++++++++++++++++++ yarn-project/kv-store/src/lmdb/store.ts | 12 +- yarn-project/kv-store/src/utils.ts | 5 +- yarn-project/merkle-tree/src/index.ts | 1 + yarn-project/merkle-tree/src/sha_256.ts | 25 ++++ 10 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 yarn-project/end-to-end/src/e2e_outbox.test.ts create mode 100644 yarn-project/merkle-tree/src/sha_256.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 53da1c2f498..3579c1e18e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -868,6 +868,20 @@ jobs: aztec_manifest_key: end-to-end <<: *defaults_e2e_test + + e2e-outbox: + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small + steps: + - *checkout + - *setup_env + - run: + name: "Test" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_outbox.test.ts + aztec_manifest_key: end-to-end + <<: *defaults_e2e_test + uniswap-trade-on-l1-from-l2: steps: - *checkout @@ -1399,6 +1413,7 @@ workflows: - e2e-inclusion-proofs-contract: *e2e_test - e2e-pending-note-hashes-contract: *e2e_test - e2e-ordering: *e2e_test + - e2e-outbox: *e2e_test - e2e-counter: *e2e_test - e2e-private-voting: *e2e_test - uniswap-trade-on-l1-from-l2: *e2e_test @@ -1463,6 +1478,7 @@ workflows: - e2e-inclusion-proofs-contract - e2e-pending-note-hashes-contract - e2e-ordering + - e2e-outbox - e2e-counter - e2e-private-voting - uniswap-trade-on-l1-from-l2 diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index aeb6588e1bf..c8ee8cfd348 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -32,6 +32,7 @@ import { Header, INITIAL_L2_BLOCK_NUM, L1_TO_L2_MSG_TREE_HEIGHT, + L2_TO_L1_MESSAGE_LENGTH, NOTE_HASH_TREE_HEIGHT, NULLIFIER_TREE_HEIGHT, NullifierLeafPreimage, @@ -44,7 +45,8 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; import { AztecKVStore } from '@aztec/kv-store'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; -import { initStoreForRollup } from '@aztec/kv-store/utils'; +import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils'; +import { SHA256, StandardTree } from '@aztec/merkle-tree'; import { AztecKVTxPool, P2P, createP2PClient } from '@aztec/p2p'; import { GlobalVariableBuilder, @@ -113,7 +115,7 @@ export class AztecNodeService implements AztecNode { const log = createDebugLogger('aztec:node'); const storeLog = createDebugLogger('aztec:node:lmdb'); const store = await initStoreForRollup( - AztecLmdbStore.open(config.dataDirectory, storeLog), + AztecLmdbStore.open(config.dataDirectory, false, storeLog), config.l1Contracts.rollupAddress, storeLog, ); @@ -426,6 +428,47 @@ export class AztecNodeService implements AztecNode { return committedDb.getSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, leafIndex); } + /** + * Returns the index of a l2ToL1Message in a ephemeral l2 to l1 data tree as well as its sibling path. + * @remarks This tree is considered ephemeral because it is created on-demand by: taking all the l2ToL1 messages + * in a single block, and then using them to make a variable depth append-only tree with these messages as leaves. + * The tree is discarded immediately after calculating what we need from it. + * @param blockNumber - The block number at which to get the data. + * @param l2ToL1Message - The l2ToL1Message get the index / sibling path for. + * @returns A tuple of the index and the sibling path of the L2ToL1Message. + */ + public async getL2ToL1MessageIndexAndSiblingPath( + blockNumber: number | 'latest', + l2ToL1Message: Fr, + ): Promise<[number, SiblingPath]> { + const block = await this.blockSource.getBlock(blockNumber === 'latest' ? await this.getBlockNumber() : blockNumber); + + if (block === undefined) { + throw new Error('Block is not defined'); + } + + const l2ToL1Messages = block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs); + + if (l2ToL1Messages.length !== L2_TO_L1_MESSAGE_LENGTH * block.body.txEffects.length) { + throw new Error('L2 to L1 Messages are not padded'); + } + + const indexOfL2ToL1Message = l2ToL1Messages.findIndex(l2ToL1MessageInBlock => + l2ToL1MessageInBlock.equals(l2ToL1Message), + ); + + if (indexOfL2ToL1Message === -1) { + throw new Error('The L2ToL1Message you are trying to prove inclusion of does not exist'); + } + + const treeHeight = Math.ceil(Math.log2(l2ToL1Messages.length)); + + const tree = new StandardTree(openTmpStore(true), new SHA256(), 'temp_outhash_sibling_path', treeHeight); + await tree.appendLeaves(l2ToL1Messages.map(l2ToL1Msg => l2ToL1Msg.toBuffer())); + + return [indexOfL2ToL1Message, await tree.getSiblingPath(BigInt(indexOfL2ToL1Message), true)]; + } + /** * Returns a sibling path for a leaf in the committed blocks tree. * @param blockNumber - The block number at which to get the data. diff --git a/yarn-project/aztec.js/src/index.ts b/yarn-project/aztec.js/src/index.ts index 1c7812aa56f..ca8bd77e378 100644 --- a/yarn-project/aztec.js/src/index.ts +++ b/yarn-project/aztec.js/src/index.ts @@ -115,6 +115,7 @@ export { merkleTreeIds, mockTx, Comparator, + SiblingPath, } from '@aztec/circuit-types'; export { NodeInfo } from '@aztec/types/interfaces'; diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts index 653636f8c8f..96cee4c34ee 100644 --- a/yarn-project/aztec/src/cli/cmds/start_archiver.ts +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -24,7 +24,7 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise const storeLog = createDebugLogger('aztec:archiver:lmdb'); const store = await initStoreForRollup( - AztecLmdbStore.open(archiverConfig.dataDirectory, storeLog), + AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog), archiverConfig.l1Contracts.rollupAddress, storeLog, ); diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index c9b7bd55d0f..72bc6a5d8a8 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -93,6 +93,20 @@ export interface AztecNode { leafIndex: bigint, ): Promise>; + /** + * Returns the index of a l2ToL1Message in a ephemeral l2 to l1 data tree as well as its sibling path. + * @remarks This tree is considered ephemeral because it is created on-demand by: taking all the l2ToL1 messages + * in a single block, and then using them to make a variable depth append-only tree with these messages as leaves. + * The tree is discarded immediately after calculating what we need from it. + * @param blockNumber - The block number at which to get the data. + * @param l2ToL1Message - The l2ToL1Message get the index / sibling path for. + * @returns A tuple of the index and the sibling path of the L2ToL1Message. + */ + getL2ToL1MessageIndexAndSiblingPath( + blockNumber: number | 'latest', + l2ToL1Message: Fr, + ): Promise<[number, SiblingPath]>; + /** * Returns a sibling path for a leaf in the committed historic blocks tree. * @param blockNumber - The block number at which to get the data. diff --git a/yarn-project/end-to-end/src/e2e_outbox.test.ts b/yarn-project/end-to-end/src/e2e_outbox.test.ts new file mode 100644 index 00000000000..dd63b6964f7 --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_outbox.test.ts @@ -0,0 +1,115 @@ +import { + AccountWalletWithPrivateKey, + AztecNode, + BatchCall, + DeployL1Contracts, + EthAddress, + Fr, + SiblingPath, + sha256, +} from '@aztec/aztec.js'; +import { SHA256 } from '@aztec/merkle-tree'; +import { TestContract } from '@aztec/noir-contracts.js'; + +import { beforeEach, describe, expect, it } from '@jest/globals'; + +import { setup } from './fixtures/utils.js'; + +// @remark - This does not test the Outbox Contract yet. All this test does is create L2 to L1 messages in a block, +// verify their existence, and produce a sibling path that is also checked for validity against the circuit produced +// out_hash in the header. +describe('E2E Outbox Tests', () => { + let teardown: () => void; + let aztecNode: AztecNode; + const merkleSha256 = new SHA256(); + let contract: TestContract; + let wallets: AccountWalletWithPrivateKey[]; + let deployL1ContractsValues: DeployL1Contracts; + + beforeEach(async () => { + ({ teardown, aztecNode, wallets, deployL1ContractsValues } = await setup(1)); + + const receipt = await TestContract.deploy(wallets[0]).send({ contractAddressSalt: Fr.ZERO }).wait(); + contract = receipt.contract; + }, 100_000); + + afterAll(() => teardown()); + + it('Inserts a new transaction with two out messages, and verifies sibling paths of both the new messages', async () => { + const [[recipient1, content1], [recipient2, content2]] = [ + [EthAddress.random(), Fr.random()], + [EthAddress.random(), Fr.random()], + ]; + + // We can't put any more l2 to L1 messages here There are a max of 2 L2 to L1 messages per transaction + const call = new BatchCall(wallets[0], [ + contract.methods.create_l2_to_l1_message_arbitrary_recipient_private(content1, recipient1).request(), + contract.methods.create_l2_to_l1_message_arbitrary_recipient_private(content2, recipient2).request(), + ]); + + // TODO (#5104): When able to guarantee multiple txs in a single block, make this populate a full tree. Right now we are + // unable to do this because in CI, for some reason, the tx's are handled in different blocks, so it is impossible + // to make a full tree of L2 -> L1 messages as we are only able to set one tx's worth of L1 -> L2 messages in a block (2 messages out of 4) + const txReceipt = await call.send().wait(); + + const block = await aztecNode.getBlock(txReceipt.blockNumber!); + + const l2ToL1Messages = block?.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs); + + expect(l2ToL1Messages?.map(l2ToL1Message => l2ToL1Message.toString())).toStrictEqual( + [makeL2ToL1Message(recipient2, content2), makeL2ToL1Message(recipient1, content1), Fr.ZERO, Fr.ZERO].map( + expectedL2ToL1Message => expectedL2ToL1Message.toString(), + ), + ); + + // For each individual message, we are using our node API to grab the index and sibling path. We expect + // the index to match the order of the block we obtained earlier. We also then use this sibling path to hash up to the root, + // verifying that the expected root obtained through the message and the sibling path match the actual root + // that was returned by the circuits in the header as out_hash. + const [index, siblingPath] = await aztecNode.getL2ToL1MessageIndexAndSiblingPath( + txReceipt.blockNumber!, + l2ToL1Messages![0], + ); + expect(siblingPath.pathSize).toBe(2); + expect(index).toBe(0); + const expectedRoot = calculateExpectedRoot(l2ToL1Messages![0], siblingPath as SiblingPath<2>, index); + expect(expectedRoot.toString('hex')).toEqual(block?.header.contentCommitment.outHash.toString('hex')); + + const [index2, siblingPath2] = await aztecNode.getL2ToL1MessageIndexAndSiblingPath( + txReceipt.blockNumber!, + l2ToL1Messages![1], + ); + expect(siblingPath2.pathSize).toBe(2); + expect(index2).toBe(1); + const expectedRoot2 = calculateExpectedRoot(l2ToL1Messages![1], siblingPath2 as SiblingPath<2>, index2); + expect(expectedRoot2.toString('hex')).toEqual(block?.header.contentCommitment.outHash.toString('hex')); + }, 360_000); + + function calculateExpectedRoot(l2ToL1Message: Fr, siblingPath: SiblingPath<2>, index: number): Buffer { + const firstLayerInput: [Buffer, Buffer] = + index & 0x1 + ? [siblingPath.toBufferArray()[0], l2ToL1Message.toBuffer()] + : [l2ToL1Message.toBuffer(), siblingPath.toBufferArray()[0]]; + const firstLayer = merkleSha256.hash(...firstLayerInput); + index /= 2; + const secondLayerInput: [Buffer, Buffer] = + index & 0x1 ? [siblingPath.toBufferArray()[1], firstLayer] : [firstLayer, siblingPath.toBufferArray()[1]]; + return merkleSha256.hash(...secondLayerInput); + } + + function makeL2ToL1Message(recipient: EthAddress, content: Fr = Fr.ZERO): Fr { + const leaf = Fr.fromBufferReduce( + sha256( + Buffer.concat([ + contract.address.toBuffer(), + new Fr(1).toBuffer(), // aztec version + recipient.toBuffer32(), + new Fr(deployL1ContractsValues.publicClient.chain.id).toBuffer(), // chain id + content.toBuffer(), + ]), + ), + ); + + return leaf; + } +}); diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index 02fc4cc787a..af949224e97 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -45,12 +45,20 @@ export class AztecLmdbStore implements AztecKVStore { * different rollup instances. * * @param path - A path on the disk to store the database. Optional + * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional * @param log - A logger to use. Optional * @returns The store */ - static open(path?: string, log = createDebugLogger('aztec:kv-store:lmdb')): AztecLmdbStore { + static open( + path?: string, + ephemeral: boolean = false, + log = createDebugLogger('aztec:kv-store:lmdb'), + ): AztecLmdbStore { log.info(`Opening LMDB database at ${path || 'temporary location'}`); - const rootDb = open({ path }); + const rootDb = open({ + path, + noSync: ephemeral, + }); return new AztecLmdbStore(rootDb); } diff --git a/yarn-project/kv-store/src/utils.ts b/yarn-project/kv-store/src/utils.ts index 7cab23f5827..e41297b9e82 100644 --- a/yarn-project/kv-store/src/utils.ts +++ b/yarn-project/kv-store/src/utils.ts @@ -34,8 +34,9 @@ export async function initStoreForRollup( /** * Opens a temporary store for testing purposes. + * @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional * @returns A new store */ -export function openTmpStore(): AztecKVStore { - return AztecLmdbStore.open(); +export function openTmpStore(ephemeral: boolean = false): AztecKVStore { + return AztecLmdbStore.open(undefined, ephemeral); } diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index ca08070bc8e..7ac5cb2dc6e 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -3,6 +3,7 @@ export * from './interfaces/indexed_tree.js'; export * from './interfaces/merkle_tree.js'; export * from './interfaces/update_only_tree.js'; export * from './pedersen.js'; +export * from './sha_256.js'; export * from './sparse_tree/sparse_tree.js'; export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tree.js'; export { StandardIndexedTreeWithAppend } from './standard_indexed_tree/test/standard_indexed_tree_with_append.js'; diff --git a/yarn-project/merkle-tree/src/sha_256.ts b/yarn-project/merkle-tree/src/sha_256.ts new file mode 100644 index 00000000000..2ed2ba5a35d --- /dev/null +++ b/yarn-project/merkle-tree/src/sha_256.ts @@ -0,0 +1,25 @@ +import { sha256 } from '@aztec/foundation/crypto'; +import { Hasher } from '@aztec/types/interfaces'; + +/** + * A helper class encapsulating SHA256 hash functionality. + * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific + * purposes. + */ +export class SHA256 implements Hasher { + /* + * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific + * purposes. + */ + public hash(lhs: Uint8Array, rhs: Uint8Array): Buffer { + return sha256(Buffer.concat([Buffer.from(lhs), Buffer.from(rhs)])); + } + + /* + * @deprecated Don't call SHA256 directly in production code. Instead, create suitably-named functions for specific + * purposes. + */ + public hashInputs(inputs: Buffer[]): Buffer { + return sha256(Buffer.concat(inputs)); + } +}