diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index fb5004af6c..1c98339480 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -90,7 +90,7 @@ jobs: -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ -o corset.tar.gz \ - https://api.github.com/repos/Consensys/corset/releases/assets/146500253 + https://api.github.com/repos/Consensys/corset/releases/assets/147491099 tar xzf corset.tar.gz mv corset $HOME echo $HOME >> $GITHUB_PATH diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/BlockCapturer.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/BlockCapturer.java new file mode 100644 index 0000000000..2d5e409ba7 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/BlockCapturer.java @@ -0,0 +1,128 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture; + +import java.util.List; + +import com.google.gson.Gson; +import net.consensys.linea.blockcapture.reapers.Reaper; +import net.consensys.linea.zktracer.ConflationAwareOperationTracer; +import net.consensys.linea.zktracer.opcode.OpCode; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Transaction; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.internal.Words; +import org.hyperledger.besu.evm.log.Log; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.evm.worldstate.WorldView; +import org.hyperledger.besu.plugin.data.BlockBody; +import org.hyperledger.besu.plugin.data.BlockHeader; + +public class BlockCapturer implements ConflationAwareOperationTracer { + /** + * The {@link Reaper} will collect all the data that will need to be mimicked to replay the block. + */ + private final Reaper reaper = new Reaper(); + /** + * This keeps a pointer to the initial state (i.e. ) to be used at the end of tracing to store the + * minimal required information to replay the conflation. + */ + private WorldUpdater worldUpdater; + + /** + * Must be called **before** any tracing activity. + * + * @param worldUpdater the state of the world before the conflation is applied + */ + public void setWorld(WorldUpdater worldUpdater) { + this.worldUpdater = worldUpdater; + } + + @Override + public void traceStartConflation(long numBlocksInConflation) {} + + @Override + public void traceEndConflation() {} + + @Override + public void traceStartBlock(BlockHeader blockHeader, BlockBody blockBody) { + this.reaper.enterBlock(blockHeader, blockBody); + } + + @Override + public void traceStartTransaction(WorldView worldView, Transaction transaction) { + this.reaper.enterTransaction(transaction); + } + + @Override + public void traceEndTransaction( + WorldView worldView, + Transaction tx, + boolean status, + Bytes output, + List logs, + long gasUsed, + long timeNs) { + this.reaper.exitTransaction(status); + } + + /** + * This method only bothers with instruction putatively accessing the state as it was at the + * beginning of the conflation. + * + * @param frame the frame + */ + @Override + public void tracePreExecution(MessageFrame frame) { + final OpCode opCode = OpCode.of(frame.getCurrentOperation().getOpcode()); + + switch (opCode) { + // These access contracts potentially existing before the conflation played out. + case EXTCODESIZE, EXTCODECOPY, EXTCODEHASH -> { + if (frame.stackSize() > 0) { + final Address target = Words.toAddress(frame.getStackItem(0)); + this.reaper.touchAddress(target); + } + } + + // SLOAD may access storage cells whose value was set before the conflation execution. + case SLOAD -> { + if (frame.stackSize() > 0) { + final Account account = frame.getWorldUpdater().get(frame.getRecipientAddress()); + final Address address = account.getAddress(); + final UInt256 key = UInt256.fromBytes(frame.getStackItem(0)); + this.reaper.touchStorage(address, key); + } + } + + // These access contracts potentially existing before the conflation played out. + case CALL, CALLCODE, DELEGATECALL, STATICCALL -> { + if (frame.stackSize() > 1) { + final Address target = Words.toAddress(frame.getStackItem(1)); + this.reaper.touchAddress(target); + } + } + } + } + + public String toJson() { + Gson gson = new Gson(); + return gson.toJson(this.reaper.collapse(this.worldUpdater)); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/AddressReaper.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/AddressReaper.java new file mode 100644 index 0000000000..9c6637b0fd --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/AddressReaper.java @@ -0,0 +1,53 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.reapers; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.hyperledger.besu.datatypes.Address; + +public class AddressReaper { + private final ArrayDeque> reaped = new ArrayDeque<>(); + + public AddressReaper() { + // “Bedrock” address set for block-level gathering. + this.reaped.addLast(new HashSet<>()); + } + + public void enterTransaction() { + this.reaped.addLast(new HashSet<>()); + } + + public void exitTransaction(boolean success) { + if (!success) { + this.reaped.removeLast(); + } + } + + public void touch(final Address... addresses) { + for (Address address : addresses) { + this.reaped.peekLast().add(address); + } + } + + public Set
collapse() { + return this.reaped.stream().flatMap(Collection::stream).collect(Collectors.toSet()); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/Reaper.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/Reaper.java new file mode 100644 index 0000000000..fe458d56c8 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/Reaper.java @@ -0,0 +1,101 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.reapers; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import net.consensys.linea.blockcapture.snapshots.AccountSnapshot; +import net.consensys.linea.blockcapture.snapshots.BlockSnapshot; +import net.consensys.linea.blockcapture.snapshots.ConflationSnapshot; +import net.consensys.linea.blockcapture.snapshots.StorageSnapshot; +import org.apache.tuweni.units.bigints.UInt256; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Transaction; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.data.BlockBody; +import org.hyperledger.besu.plugin.data.BlockHeader; + +/** + * The Reaper collect all the information from the state that will be accessed during the execution + * of a conflation. + * + *

This data can than be collapsed into a “replay” ({@link ConflationSnapshot}), i.e. the minimal + * required information to replay a conflation as if it were executed on the blockchain. + */ +public class Reaper { + /** Collect the reads from the state */ + private final StorageReaper storage = new StorageReaper(); + /** Collect the addresses read from the state */ + private final AddressReaper addresses = new AddressReaper(); + /** Collect the blocks within a conflation */ + private final List blocks = new ArrayList<>(); + + public void enterBlock(final BlockHeader header, final BlockBody body) { + this.blocks.add( + BlockSnapshot.of((org.hyperledger.besu.ethereum.core.BlockHeader) header, body)); + this.addresses.touch(header.getCoinbase()); + } + + public void enterTransaction(Transaction tx) { + this.storage.enterTransaction(); + this.addresses.enterTransaction(); + + this.touchAddress(tx.getSender()); + tx.getTo().ifPresent(this::touchAddress); + } + + public void exitTransaction(boolean success) { + this.storage.exitTransaction(success); + this.addresses.exitTransaction(success); + } + + public void touchAddress(final Address address) { + this.addresses.touch(address); + } + + public void touchStorage(final Address address, final UInt256 key) { + this.storage.touch(address, key); + } + + /** + * Uniquify and solidify the accumulated data, then return a {@link ConflationSnapshot}, which + * contains the smallest dataset required to exactly replay the conflation within a test framework + * without requiring access to the whole state. + * + * @param world the state before the conflation execution + * @return a minimal set of information required to replay the conflation within a test framework + */ + public ConflationSnapshot collapse(final WorldUpdater world) { + final List initialAccounts = + this.addresses.collapse().stream() + .flatMap(a -> AccountSnapshot.from(a, world).stream()) + .toList(); + + final List initialStorage = new ArrayList<>(); + for (Map.Entry> e : this.storage.collapse().entrySet()) { + final Address address = e.getKey(); + + e.getValue().stream() + .flatMap(key -> StorageSnapshot.from(address, key, world).stream()) + .forEach(initialStorage::add); + } + + return new ConflationSnapshot(this.blocks, initialAccounts, initialStorage); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/StorageReaper.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/StorageReaper.java new file mode 100644 index 0000000000..13e50f8ee7 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/StorageReaper.java @@ -0,0 +1,63 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.reapers; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.tuweni.units.bigints.UInt256; +import org.hyperledger.besu.datatypes.Address; + +/** + * This object gathers all non-reversed accesses to storage values during the execution of a + * conflation, then collapse them into a single mapping of the initial values in these slots. + */ +public class StorageReaper { + private final Deque>> transientStates = new ArrayDeque<>(); + + public void enterTransaction() { + this.transientStates.addLast(new HashMap<>()); + } + + public void exitTransaction(boolean success) { + if (!success) { + this.transientStates.removeLast(); + } + } + + public void touch(final Address address, final UInt256 key) { + this.transientStates.peekLast().computeIfAbsent(address, k -> new HashSet<>()).add(key); + } + + public Map> collapse() { + final Map> r = new HashMap<>(); + + for (var txEntry : this.transientStates) { + for (Map.Entry> addressKeys : txEntry.entrySet()) { + final Address address = addressKeys.getKey(); + + // Use computeIfAbsent instead of put, as we only want to capture the **first** read. + r.computeIfAbsent(address, k -> new HashSet<>()).addAll(addressKeys.getValue()); + } + } + + return r; + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/AccessListEntrySnapshot.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/AccessListEntrySnapshot.java new file mode 100644 index 0000000000..edd26f2544 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/AccessListEntrySnapshot.java @@ -0,0 +1,26 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.snapshots; + +import java.util.List; + +import org.hyperledger.besu.datatypes.AccessListEntry; + +public record AccessListEntrySnapshot(String address, List storageKeys) { + public static AccessListEntrySnapshot from(AccessListEntry e) { + return new AccessListEntrySnapshot(e.getAddressString(), e.getStorageKeysString()); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/AccountSnapshot.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/AccountSnapshot.java new file mode 100644 index 0000000000..f05d0be748 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/AccountSnapshot.java @@ -0,0 +1,34 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.snapshots; + +import java.util.Optional; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; + +public record AccountSnapshot(String address, long nonce, String balance, String code) { + public static Optional from(Address address, WorldUpdater world) { + return Optional.ofNullable(world.get(address)) + .map( + account -> + new AccountSnapshot( + address.toHexString(), + account.getNonce(), + account.getBalance().toHexString(), + account.getCode().toHexString())); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/BlockHeaderSnapshot.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/BlockHeaderSnapshot.java new file mode 100644 index 0000000000..2bfd29271f --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/BlockHeaderSnapshot.java @@ -0,0 +1,94 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.snapshots; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Quantity; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; +import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; +import org.hyperledger.besu.evm.log.LogsBloomFilter; + +public record BlockHeaderSnapshot( + String parentHash, + String ommersHash, + String coinbase, + String stateRoot, + String transactionRoot, + String receiptsRoot, + String logsBloom, + String difficulty, + long number, + long gasLimit, + long gasUsed, + long timestamp, + String extraData, + String mixHashOrPrevRandao, + long nonce, + Optional baseFee) { + public static BlockHeaderSnapshot from(BlockHeader header) { + return new BlockHeaderSnapshot( + header.getParentHash().toHexString(), + header.getOmmersHash().toHexString(), + header.getCoinbase().toHexString(), + header.getStateRoot().toHexString(), + header.getTransactionsRoot().toHexString(), + header.getReceiptsRoot().toHexString(), + header.getLogsBloom().toHexString(), + header.getDifficulty().toHexString(), + header.getNumber(), + header.getGasLimit(), + header.getGasUsed(), + header.getTimestamp(), + header.getExtraData().toHexString(), + header.getMixHashOrPrevRandao().toHexString(), + header.getNonce(), + header.getBaseFee().map(Quantity::toHexString)); + } + + public BlockHeader toBlockHeader() { + final BlockHeaderBuilder builder = + BlockHeaderBuilder.create() + .parentHash(Hash.fromHexString(this.parentHash)) + .ommersHash(Hash.fromHexString(this.ommersHash)) + .coinbase(Address.fromHexString(this.coinbase)) + .stateRoot(Hash.fromHexString(this.stateRoot)) + .transactionsRoot(Hash.fromHexString(this.transactionRoot)) + .receiptsRoot(Hash.fromHexString(this.receiptsRoot)) + .logsBloom(LogsBloomFilter.fromHexString(this.logsBloom)) + .difficulty(Difficulty.fromHexString(this.difficulty)) + .number(this.number) + .gasLimit(this.gasLimit) + .gasUsed(this.gasUsed) + .timestamp(this.timestamp) + .extraData(Bytes.fromHexString(this.extraData)) + .mixHash(Hash.fromHexString(this.mixHashOrPrevRandao)) + .prevRandao(Bytes32.fromHexString(this.mixHashOrPrevRandao)) + .nonce(this.nonce) + .blockHeaderFunctions(new MainnetBlockHeaderFunctions()); + + this.baseFee.ifPresent(baseFee -> builder.baseFee(Wei.fromHexString(baseFee))); + + return builder.buildBlockHeader(); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/BlockSnapshot.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/BlockSnapshot.java new file mode 100644 index 0000000000..2bfc693030 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/BlockSnapshot.java @@ -0,0 +1,30 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.snapshots; + +import java.util.List; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.plugin.data.BlockBody; + +public record BlockSnapshot(BlockHeaderSnapshot header, List txs) { + public static BlockSnapshot of(final BlockHeader header, final BlockBody body) { + return new BlockSnapshot( + BlockHeaderSnapshot.from(header), + body.getTransactions().stream().map(t -> TransactionSnapshot.of((Transaction) t)).toList()); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/ConflationSnapshot.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/ConflationSnapshot.java new file mode 100644 index 0000000000..c7162766eb --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/ConflationSnapshot.java @@ -0,0 +1,29 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.snapshots; + +import java.util.List; + +/** + * Contain the minimal set of information to replay a conflation as a unit test without requiring + * access to the whole state. + * + * @param blocks the blocks within the conflation + * @param accounts the accounts whose state will be read during the conflation execution + * @param storage storage cells that will be accessed during the conflation execution + */ +public record ConflationSnapshot( + List blocks, List accounts, List storage) {} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/StorageSnapshot.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/StorageSnapshot.java new file mode 100644 index 0000000000..9ab10db789 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/StorageSnapshot.java @@ -0,0 +1,35 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.snapshots; + +import java.util.Optional; + +import org.apache.tuweni.units.bigints.UInt256; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; + +public record StorageSnapshot(String address, String key, String value) { + public static Optional from( + Address address, UInt256 key, final WorldUpdater world) { + return Optional.ofNullable(world.get(address)) + .map( + account -> + new StorageSnapshot( + address.toHexString(), + key.toHexString(), + account.getStorageValue(key).toHexString())); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/TransactionSnapshot.java b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/TransactionSnapshot.java new file mode 100644 index 0000000000..3e999411fa --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/TransactionSnapshot.java @@ -0,0 +1,122 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.blockcapture.snapshots; + +import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_BASE; +import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_MIN; +import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_UNPROTECTED_V_BASE; + +import java.math.BigInteger; +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.DelegatingBytes; +import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; +import org.hyperledger.besu.datatypes.AccessListEntry; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Quantity; +import org.hyperledger.besu.datatypes.TransactionType; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.evm.internal.Words; + +public record TransactionSnapshot( + String r, + String s, + String v, + TransactionType type, + String sender, + Optional to, + long nonce, + String value, + String payload, + Optional gasPrice, + Optional maxPriorityFeePerGas, + Optional maxFeePerGas, + Optional maxFeePerBlobGas, + long gasLimit, + BigInteger chainId, + Optional> accessList) { + public static final BigInteger CHAIN_ID = BigInteger.valueOf(1337); + + public static TransactionSnapshot of(Transaction tx) { + return new TransactionSnapshot( + tx.getS().toString(16), + tx.getR().toString(16), + tx.getType() == TransactionType.FRONTIER + ? tx.getV().toString(16) + : tx.getYParity().toString(16), + tx.getType(), + tx.getSender().toHexString(), + tx.getTo().map(DelegatingBytes::toHexString), + tx.getNonce(), + tx.getValue().toHexString(), + tx.getPayload().toHexString(), + tx.getGasPrice().map(Quantity::toHexString), + tx.getMaxPriorityFeePerGas().map(Quantity::toHexString), + tx.getMaxFeePerGas().map(Quantity::toHexString), + tx.getMaxFeePerBlobGas().map(Quantity::toHexString), + tx.getGasLimit(), + tx.getChainId().orElse(CHAIN_ID), + tx.getAccessList().map(l -> l.stream().map(AccessListEntrySnapshot::from).toList())); + } + + public Transaction toTransaction() { + BigInteger r = Bytes.fromHexStringLenient(this.r).toUnsignedBigInteger(); + BigInteger s = Bytes.fromHexStringLenient(this.s).toUnsignedBigInteger(); + BigInteger v = Bytes.fromHexStringLenient(this.v).toUnsignedBigInteger(); + + if (this.type == TransactionType.FRONTIER) { + if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { + v = v.subtract(REPLAY_PROTECTED_V_BASE).subtract(chainId.multiply(BigInteger.TWO)); + } else { + v = v.subtract(REPLAY_UNPROTECTED_V_BASE); + } + } + + final var tx = + Transaction.builder() + .type(this.type) + .sender(Address.fromHexString(this.sender)) + .nonce(this.nonce) + .value(Wei.fromHexString(this.value)) + .payload(Bytes.fromHexString(this.payload)) + .chainId(this.chainId) + .gasLimit(this.gasLimit) + .signature( + SignatureAlgorithmFactory.getInstance().createSignature(r, s, v.byteValueExact())); + + this.to.ifPresent(to -> tx.to(Words.toAddress(Bytes.fromHexString(to)))); + this.gasPrice.ifPresent(gasPrice -> tx.gasPrice(Wei.fromHexString(gasPrice))); + this.maxPriorityFeePerGas.ifPresent( + maxPriorityFeePerGas -> tx.maxPriorityFeePerGas(Wei.fromHexString(maxPriorityFeePerGas))); + this.maxFeePerGas.ifPresent(maxFeePerGas -> tx.maxFeePerGas(Wei.fromHexString(maxFeePerGas))); + this.maxFeePerBlobGas.ifPresent( + maxFeePerBlobGas -> tx.maxFeePerBlobGas(Wei.fromHexString(maxFeePerBlobGas))); + this.accessList.ifPresent( + l -> + tx.accessList( + l.stream() + .map( + e -> + AccessListEntry.createAccessListEntry( + Address.fromHexString(e.address()), e.storageKeys())) + .toList())); + + return tx.build(); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/capture/Capture.java b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/Capture.java new file mode 100644 index 0000000000..7277bdd99b --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/Capture.java @@ -0,0 +1,21 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.rpc.capture; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** FileTrace represents an execution trace. */ +public record Capture(@JsonProperty("capture") String capture) {} diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureEndpointServicePlugin.java b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureEndpointServicePlugin.java new file mode 100644 index 0000000000..b7d7bb78ce --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureEndpointServicePlugin.java @@ -0,0 +1,72 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.rpc.capture; + +import java.util.Optional; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.AbstractLineaRequiredPlugin; +import net.consensys.linea.zktracer.opcode.OpCodes; +import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.services.RpcEndpointService; + +/** + * Registers RPC endpoints .This class provides an RPC endpoint named + * 'generateConflatedTracesToFileV0' under the 'rollup' namespace. It uses {@link CaptureToFile} to + * generate conflated file traces. This class provides an RPC endpoint named + * 'generateConflatedTracesToFileV0' under the 'rollup' namespace. + */ +@AutoService(BesuPlugin.class) +@Slf4j +public class CaptureEndpointServicePlugin extends AbstractLineaRequiredPlugin { + + /** + * Register the RPC service. + * + * @param context the BesuContext to be used. + */ + @Override + public void doRegister(final BesuContext context) { + CaptureToFile method = new CaptureToFile(context); + + Optional service = context.getService(RpcEndpointService.class); + createAndRegister( + method, + service.orElseThrow( + () -> + new RuntimeException("Failed to obtain RpcEndpointService from the BesuContext."))); + } + + /** + * Create and register the RPC service. + * + * @param method the RollupGenerateConflatedTracesToFileV0 method to be used. + * @param rpcEndpointService the RpcEndpointService to be registered. + */ + private void createAndRegister( + final CaptureToFile method, final RpcEndpointService rpcEndpointService) { + rpcEndpointService.registerRPCEndpoint( + method.getNamespace(), method.getName(), method::execute); + } + + /** Start the RPC service. This method loads the OpCodes. */ + @Override + public void start() { + OpCodes.load(); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureParams.java b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureParams.java new file mode 100644 index 0000000000..a9973d99ee --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureParams.java @@ -0,0 +1,43 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.rpc.capture; + +import java.security.InvalidParameterException; + +/** Holds needed parameters for sending an execution trace generation request. */ +@SuppressWarnings("unused") +public record CaptureParams(long fromBlock, long toBlock) { + private static final int EXPECTED_PARAMS_SIZE = 2; + + /** + * Parses a list of params to a {@link CaptureParams} object. + * + * @param params an array of parameters. + * @return a parsed {@link CaptureParams} object.. + */ + public static CaptureParams createTraceParams(final Object[] params) { + // validate params size + if (params.length != EXPECTED_PARAMS_SIZE) { + throw new InvalidParameterException( + String.format("Expected %d parameters but got %d", EXPECTED_PARAMS_SIZE, params.length)); + } + + long fromBlock = Long.parseLong(params[0].toString()); + long toBlock = Long.parseLong(params[1].toString()); + + return new CaptureParams(fromBlock, toBlock); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureToFile.java b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureToFile.java new file mode 100644 index 0000000000..5bb4630f4e --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/capture/CaptureToFile.java @@ -0,0 +1,86 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.rpc.capture; + +import com.google.common.base.Stopwatch; +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.blockcapture.BlockCapturer; +import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.services.TraceService; +import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; + +/** + * Sets up an RPC endpoint for generating conflated file trace. This class provides an RPC endpoint + * named 'generateConflatedTracesToFileV0' under the 'rollup' namespace. When this endpoint is + * called, it triggers the execution of the 'execute' method, which generates conflated file traces + * based on the provided request parameters and writes them to a file. + */ +@Slf4j +public class CaptureToFile { + private final BesuContext besuContext; + private TraceService traceService; + + public CaptureToFile(final BesuContext besuContext) { + this.besuContext = besuContext; + } + + public String getNamespace() { + return "rollup"; + } + + public String getName() { + return "captureConflation"; + } + + /** + * Handles execution traces generation logic. + * + * @param request holds parameters of the RPC request. + * @return an execution file trace. + */ + public Capture execute(final PluginRpcRequest request) { + if (this.traceService == null) { + this.traceService = getTraceService(); + } + + CaptureParams params = CaptureParams.createTraceParams(request.getParams()); + final long fromBlock = params.fromBlock(); + final long toBlock = params.toBlock(); + final BlockCapturer tracer = new BlockCapturer(); + + Stopwatch sw = Stopwatch.createStarted(); + traceService.trace( + fromBlock, + toBlock, + worldStateBeforeTracing -> { + tracer.setWorld(worldStateBeforeTracing); + tracer.traceStartConflation(toBlock - fromBlock + 1); + }, + worldStateAfterTracing -> tracer.traceEndConflation(), + tracer); + log.info("[CAPTURE] capture for {}-{} computed in {}", fromBlock, toBlock, sw); + return new Capture(tracer.toJson()); + } + + private TraceService getTraceService() { + return this.besuContext + .getService(TraceService.class) + .orElseThrow( + () -> + new RuntimeException( + "Unable to find trace service. Please ensure TraceService is registered.")); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/counters/RollupRpcEndpointServicePlugin.java b/arithmetization/src/main/java/net/consensys/linea/rpc/counters/CountersEndpointServicePlugin.java similarity index 96% rename from arithmetization/src/main/java/net/consensys/linea/rpc/counters/RollupRpcEndpointServicePlugin.java rename to arithmetization/src/main/java/net/consensys/linea/rpc/counters/CountersEndpointServicePlugin.java index d5bf2ad282..8496e8eaf6 100644 --- a/arithmetization/src/main/java/net/consensys/linea/rpc/counters/RollupRpcEndpointServicePlugin.java +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/counters/CountersEndpointServicePlugin.java @@ -33,7 +33,7 @@ * RollupGenerateCountersV0} */ @AutoService(BesuPlugin.class) -public class RollupRpcEndpointServicePlugin extends AbstractLineaRequiredPlugin { +public class CountersEndpointServicePlugin extends AbstractLineaRequiredPlugin { /** * Register the RPC service. diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/tracegeneration/RollupRpcEndpointServicePlugin.java b/arithmetization/src/main/java/net/consensys/linea/rpc/tracegeneration/TracesEndpointServicePlugin.java similarity index 96% rename from arithmetization/src/main/java/net/consensys/linea/rpc/tracegeneration/RollupRpcEndpointServicePlugin.java rename to arithmetization/src/main/java/net/consensys/linea/rpc/tracegeneration/TracesEndpointServicePlugin.java index c8a7885a98..212b8b520b 100644 --- a/arithmetization/src/main/java/net/consensys/linea/rpc/tracegeneration/RollupRpcEndpointServicePlugin.java +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/tracegeneration/TracesEndpointServicePlugin.java @@ -33,7 +33,7 @@ */ @AutoService(BesuPlugin.class) @Slf4j -public class RollupRpcEndpointServicePlugin extends AbstractLineaRequiredPlugin { +public class TracesEndpointServicePlugin extends AbstractLineaRequiredPlugin { /** * Register the RPC service. diff --git a/arithmetization/src/main/java/net/consensys/linea/zktracer/ZkBlockAwareOperationTracer.java b/arithmetization/src/main/java/net/consensys/linea/zktracer/ConflationAwareOperationTracer.java similarity index 65% rename from arithmetization/src/main/java/net/consensys/linea/zktracer/ZkBlockAwareOperationTracer.java rename to arithmetization/src/main/java/net/consensys/linea/zktracer/ConflationAwareOperationTracer.java index 7de18f02f4..b9141d6921 100644 --- a/arithmetization/src/main/java/net/consensys/linea/zktracer/ZkBlockAwareOperationTracer.java +++ b/arithmetization/src/main/java/net/consensys/linea/zktracer/ConflationAwareOperationTracer.java @@ -15,20 +15,13 @@ package net.consensys.linea.zktracer; -import java.nio.file.Path; -import java.util.List; - -import org.apache.tuweni.bytes.Bytes; -import org.hyperledger.besu.datatypes.Transaction; -import org.hyperledger.besu.evm.log.Log; -import org.hyperledger.besu.evm.worldstate.WorldView; import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer; /** * An extended operation tracer that can trace the start and end of a number of blocks in * conflation. */ -public interface ZkBlockAwareOperationTracer extends BlockAwareOperationTracer { +public interface ConflationAwareOperationTracer extends BlockAwareOperationTracer { /** * Trace the start of conflation for a number of blocks. @@ -39,17 +32,4 @@ public interface ZkBlockAwareOperationTracer extends BlockAwareOperationTracer { /** Trace the end of conflation for a number of blocks. */ void traceEndConflation(); - - void writeToFile(final Path filename); - - void traceStartTransaction(WorldView worldView, Transaction transaction); - - void traceEndTransaction( - WorldView worldView, - Transaction tx, - boolean status, - Bytes output, - List logs, - long gasUsed, - long timeNs); } diff --git a/arithmetization/src/main/java/net/consensys/linea/zktracer/ZkTracer.java b/arithmetization/src/main/java/net/consensys/linea/zktracer/ZkTracer.java index 063fab6b57..9fdbba98cf 100644 --- a/arithmetization/src/main/java/net/consensys/linea/zktracer/ZkTracer.java +++ b/arithmetization/src/main/java/net/consensys/linea/zktracer/ZkTracer.java @@ -49,7 +49,7 @@ import org.hyperledger.besu.plugin.data.ProcessableBlockHeader; @Slf4j -public class ZkTracer implements ZkBlockAwareOperationTracer { +public class ZkTracer implements ConflationAwareOperationTracer { /** The {@link GasCalculator} used in this version of the arithmetization */ public static final GasCalculator gasCalculator = new LondonGasCalculator(); @@ -92,7 +92,6 @@ public Path writeToTmpFile(final Path rootDir) { } } - @Override public void writeToFile(final Path filename) { final List modules = this.hub.getModulesToTrace(); final List traceMap = @@ -153,7 +152,7 @@ public void traceEndBlock(final BlockHeader blockHeader, final BlockBody blockBo } @Override - public void traceStartTransaction(WorldView worldView, Transaction transaction) { + public void tracePrepareTransaction(WorldView worldView, Transaction transaction) { hashOfLastTransactionTraced = transaction.getHash(); this.hub.traceStartTx(worldView, transaction); } diff --git a/arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/Exceptions.java b/arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/Exceptions.java index b89586fdf0..7e3472a311 100644 --- a/arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/Exceptions.java +++ b/arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/Exceptions.java @@ -32,7 +32,7 @@ @Accessors(fluent = true) public final class Exceptions { private static final byte EIP_3541_MARKER = (byte) 0xEF; - private static final int MAX_CODE_SIZE = 24576; + public static final int MAX_CODE_SIZE = 24576; private final Hub hub; diff --git a/arithmetization/src/test/java/net/consensys/linea/zktracer/ReplaysTests.java b/arithmetization/src/test/java/net/consensys/linea/zktracer/ReplaysTests.java new file mode 100644 index 0000000000..0afcb0c002 --- /dev/null +++ b/arithmetization/src/test/java/net/consensys/linea/zktracer/ReplaysTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.zktracer; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.zip.GZIPInputStream; + +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.zktracer.testing.ToyExecutionEnvironment; +import org.junit.jupiter.api.Test; + +/** + * Replays are captured on a fully (not snapshot) synchronized Besu node running the plugin: + * + * 

{@code
+ * curl -X POST 'http://localhost:8545'
+ * --data '{
+ *    "jsonrpc":"2.0",
+ *    "method":"rollup_captureConflation",
+ *    "params":["296519", "296521"], "id":"1"
+ *  }'
+ * | jq '.result.capture' -r
+ * | gzip > arithmetization/src/test/resources/replays/my-test-case.json.gz
+ * }
+ */ +@Slf4j +public class ReplaysTests { + /** + * Loads a .json or .json.gz replay file generated by the {@link + * net.consensys.linea.blockcapture.BlockCapturer} and execute it as a test. + * + * @param filename the file in resources/replays/ containing the replay + */ + public static void replay(String filename) { + final InputStream fileStream = + ReplaysTests.class.getClassLoader().getResourceAsStream("replays/%s".formatted(filename)); + if (fileStream == null) { + fail("unable to find %s in replay resources".formatted(filename)); + } + + final InputStream stream; + try { + stream = filename.toLowerCase().endsWith("gz") ? new GZIPInputStream(fileStream) : fileStream; + } catch (IOException e) { + log.error("while loading {}: {}", filename, e.getMessage()); + throw new RuntimeException(e); + } + ToyExecutionEnvironment.builder() + .build() + .replay(new BufferedReader(new InputStreamReader(stream))); + } + + @Test + void traceTxStartNotTheSameAsTxPrepare() { + replay("start-vs-prepare-tx.json.gz"); + } +} diff --git a/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyExecutionEnvironment.java b/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyExecutionEnvironment.java index 561a66c94b..20b688b17a 100644 --- a/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyExecutionEnvironment.java +++ b/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyExecutionEnvironment.java @@ -19,18 +19,24 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; +import java.io.Reader; import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.function.Consumer; +import com.google.gson.Gson; import lombok.Builder; import lombok.RequiredArgsConstructor; import lombok.Singular; import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.blockcapture.snapshots.BlockSnapshot; +import net.consensys.linea.blockcapture.snapshots.ConflationSnapshot; +import net.consensys.linea.blockcapture.snapshots.TransactionSnapshot; import net.consensys.linea.corset.CorsetValidator; import net.consensys.linea.zktracer.ZkTracer; +import net.consensys.linea.zktracer.module.hub.Exceptions; import org.apache.tuweni.bytes.Bytes; import org.hyperledger.besu.crypto.SECP256K1; import org.hyperledger.besu.datatypes.*; @@ -73,6 +79,7 @@ public class ToyExecutionEnvironment { private final ToyWorld toyWorld; private final EVM evm; + @Builder.Default private BigInteger chainId = CHAIN_ID; @Singular private final List transactions; /** @@ -95,10 +102,10 @@ public static EVM defaultEvm() { return MainnetEVMs.london(EvmConfiguration.DEFAULT); } - public static void checkTracer(ZkTracer tracer) { + public void checkTracer() { try { final Path traceFile = Files.createTempFile(null, ".lt"); - tracer.writeToFile(traceFile); + this.tracer.writeToFile(traceFile); log.info("trace written to `{}`", traceFile); assertThat(corsetValidator.validate(traceFile).isValid()).isTrue(); } catch (IOException e) { @@ -107,13 +114,80 @@ public static void checkTracer(ZkTracer tracer) { } public void run() { - execute(); - checkTracer(this.tracer); + this.execute(); + this.checkTracer(); + } + + /** + * Given a file containing the JSON serialization of a {@link ConflationSnapshot}, loads it, + * updates this's state to mirror it, and replays it. + * + * @param replayFile the file containing the conflation + */ + public void replay(final Reader replayFile) { + Gson gson = new Gson(); + ConflationSnapshot conflation; + try { + conflation = gson.fromJson(replayFile, ConflationSnapshot.class); + } catch (Exception e) { + log.error(e.getMessage()); + return; + } + this.executeFrom(conflation); + this.checkTracer(); + } + + /** + * Loads the states and the conflation defined in a {@link ConflationSnapshot}, mimick the + * accounts, storage and blocks state as it was on the blockchain before the conflation played + * out, then execute and check it. + * + * @param conflation the conflation to replay + */ + private void executeFrom(final ConflationSnapshot conflation) { + final ToyWorld overridenToyWorld = ToyWorld.of(conflation); + for (BlockSnapshot blockSnapshot : conflation.blocks()) { + for (TransactionSnapshot tx : blockSnapshot.txs()) { + this.chainId = tx.chainId(); + } + } + final MainnetTransactionProcessor transactionProcessor = getMainnetTransactionProcessor(); + + tracer.traceStartConflation(conflation.blocks().size()); + for (BlockSnapshot blockSnapshot : conflation.blocks()) { + BlockHeader header = blockSnapshot.header().toBlockHeader(); + BlockBody body = + new BlockBody( + blockSnapshot.txs().stream().map(TransactionSnapshot::toTransaction).toList(), + new ArrayList<>()); + tracer.traceStartBlock(header, body); + + for (Transaction tx : body.getTransactions()) { + final TransactionProcessingResult result = + transactionProcessor.processTransaction( + null, + overridenToyWorld.updater(), + (ProcessableBlockHeader) header, + tx, + header.getCoinbase(), + tracer, + blockId -> { + throw new RuntimeException("Block hash lookup not yet supported"); + }, + false, + Wei.ZERO); + } + tracer.traceEndBlock(header, body); + } + tracer.traceEndConflation(); } private void execute() { BlockHeader header = - BlockHeaderBuilder.createDefault().baseFee(DEFAULT_BASE_FEE).buildBlockHeader(); + BlockHeaderBuilder.createDefault() + .baseFee(DEFAULT_BASE_FEE) + .coinbase(minerAddress) + .buildBlockHeader(); BlockBody mockBlockBody = new BlockBody(transactions, new ArrayList<>()); final MainnetTransactionProcessor transactionProcessor = getMainnetTransactionProcessor(); @@ -128,7 +202,7 @@ private void execute() { toyWorld.updater(), (ProcessableBlockHeader) header, tx, - minerAddress, + header.getCoinbase(), tracer, blockId -> { throw new RuntimeException("Block hash lookup not yet supported"); @@ -139,7 +213,6 @@ private void execute() { this.testValidator.accept(result); this.zkTracerValidator.accept(tracer); } - tracer.traceEndBlock(header, mockBlockBody); tracer.traceEndConflation(); } @@ -156,9 +229,11 @@ private MainnetTransactionProcessor getMainnetTransactionProcessor() { new TransactionValidatorFactory( gasCalculator, new LondonTargetingGasLimitCalculator(0L, new LondonFeeMarket(0, Optional.empty())), + new LondonFeeMarket(0L), false, - Optional.of(CHAIN_ID), - Set.of(TransactionType.FRONTIER, TransactionType.ACCESS_LIST, TransactionType.EIP1559)), + Optional.of(this.chainId), + Set.of(TransactionType.FRONTIER, TransactionType.ACCESS_LIST, TransactionType.EIP1559), + Exceptions.MAX_CODE_SIZE), contractCreationProcessor, messageCallProcessor, true, diff --git a/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyWorld.java b/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyWorld.java index 687f44efe0..b2b291526b 100644 --- a/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyWorld.java +++ b/arithmetization/src/test/java/net/consensys/linea/zktracer/testing/ToyWorld.java @@ -25,11 +25,16 @@ import lombok.Builder; import lombok.Getter; import lombok.Singular; +import net.consensys.linea.blockcapture.snapshots.AccountSnapshot; +import net.consensys.linea.blockcapture.snapshots.ConflationSnapshot; +import net.consensys.linea.blockcapture.snapshots.StorageSnapshot; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.internal.Words; import org.hyperledger.besu.evm.worldstate.WorldUpdater; public class ToyWorld implements WorldUpdater { @@ -56,6 +61,28 @@ public static ToyWorld empty() { return builder().build(); } + public static ToyWorld of(final ConflationSnapshot conflation) { + final ToyWorldBuilder protoWorld = builder(); + for (AccountSnapshot account : conflation.accounts()) { + protoWorld.account( + ToyAccount.builder() + .address(Words.toAddress(Address.fromHexString(account.address()))) + .nonce(account.nonce()) + .balance(Wei.fromHexString(account.balance())) + .code(Bytes.fromHexString(account.code())) + .build()); + } + final ToyWorld world = protoWorld.build(); + + for (StorageSnapshot s : conflation.storage()) { + world + .getAccount(Words.toAddress(Bytes.fromHexString(s.address()))) + .setStorageValue(UInt256.fromHexString(s.key()), UInt256.fromHexString(s.value())); + } + + return world; + } + @Override public WorldUpdater updater() { return new ToyWorld(this); diff --git a/arithmetization/src/test/resources/replays/start-vs-prepare-tx.json.gz b/arithmetization/src/test/resources/replays/start-vs-prepare-tx.json.gz new file mode 100644 index 0000000000..5937cbbccc Binary files /dev/null and b/arithmetization/src/test/resources/replays/start-vs-prepare-tx.json.gz differ