feat: mechanics to capture conflations & replay them as test cases

.github/workflows/gradle.yml

@@ -90,7 +90,7 @@ jobs:
           -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           -o corset.tar.gz \
-          https://api.github.com/repos/Consensys/corset/releases/assets/146500253
+          https://api.github.com/repos/Consensys/corset/releases/assets/147491099
           tar xzf corset.tar.gz
           mv corset $HOME
           echo $HOME >> $GITHUB_PATH

arithmetization/src/main/java/net/consensys/linea/blockcapture/BlockCapturer.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package net.consensys.linea.blockcapture;
+
+import java.util.List;
+
+import com.google.gson.Gson;
+import net.consensys.linea.blockcapture.reapers.Reaper;
+import net.consensys.linea.zktracer.ConflationAwareOperationTracer;
+import net.consensys.linea.zktracer.opcode.OpCode;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.units.bigints.UInt256;
+import org.hyperledger.besu.datatypes.Address;
+import org.hyperledger.besu.datatypes.Transaction;
+import org.hyperledger.besu.evm.account.Account;
+import org.hyperledger.besu.evm.frame.MessageFrame;
+import org.hyperledger.besu.evm.internal.Words;
+import org.hyperledger.besu.evm.log.Log;
+import org.hyperledger.besu.evm.worldstate.WorldUpdater;
+import org.hyperledger.besu.evm.worldstate.WorldView;
+import org.hyperledger.besu.plugin.data.BlockBody;
+import org.hyperledger.besu.plugin.data.BlockHeader;
+
+public class BlockCapturer implements ConflationAwareOperationTracer {
+  /**
+   * The {@link Reaper} will collect all the data that will need to be mimicked to replay the block.
+   */
+  private final Reaper reaper = new Reaper();
+  /**
+   * This keeps a pointer to the initial state (i.e. ) to be used at the end of tracing to store the
+   * minimal required information to replay the conflation.
+   */
+  private WorldUpdater worldUpdater;
+
+  /**
+   * Must be called **before** any tracing activity.
+   *
+   * @param worldUpdater the state of the world before the conflation is applied
+   */
+  public void setWorld(WorldUpdater worldUpdater) {
+    this.worldUpdater = worldUpdater;
+  }
+
+  @Override
+  public void traceStartConflation(long numBlocksInConflation) {}
+
+  @Override
+  public void traceEndConflation() {}
+
+  @Override
+  public void traceStartBlock(BlockHeader blockHeader, BlockBody blockBody) {
+    this.reaper.enterBlock(blockHeader, blockBody);
+  }
+
+  @Override
+  public void traceStartTransaction(WorldView worldView, Transaction transaction) {
+    this.reaper.enterTransaction(transaction);
+  }
+
+  @Override
+  public void traceEndTransaction(
+      WorldView worldView,
+      Transaction tx,
+      boolean status,
+      Bytes output,
+      List<Log> logs,
+      long gasUsed,
+      long timeNs) {
+    this.reaper.exitTransaction(status);
+  }
+
+  /**
+   * This method only bothers with instruction putatively accessing the state as it was at the
+   * beginning of the conflation.
+   *
+   * @param frame the frame
+   */
+  @Override
+  public void tracePreExecution(MessageFrame frame) {
+    final OpCode opCode = OpCode.of(frame.getCurrentOperation().getOpcode());
+
+    switch (opCode) {
+        // These access contracts potentially existing before the conflation played out.
+      case EXTCODESIZE, EXTCODECOPY, EXTCODEHASH -> {
+        if (frame.stackSize() > 0) {
+          final Address target = Words.toAddress(frame.getStackItem(0));
+          this.reaper.touchAddress(target);
+        }
+      }
+
+        // SLOAD may access storage cells whose value was set before the conflation execution.
+      case SLOAD -> {
+        if (frame.stackSize() > 0) {
+          final Account account = frame.getWorldUpdater().get(frame.getRecipientAddress());
+          final Address address = account.getAddress();
+          final UInt256 key = UInt256.fromBytes(frame.getStackItem(0));
+          this.reaper.touchStorage(address, key);
+        }
+      }
+
+        // These access contracts potentially existing before the conflation played out.
+      case CALL, CALLCODE, DELEGATECALL, STATICCALL -> {
+        if (frame.stackSize() > 1) {
+          final Address target = Words.toAddress(frame.getStackItem(1));
+          this.reaper.touchAddress(target);
+        }
+      }
+    }
+  }
+
+  public String toJson() {
+    Gson gson = new Gson();
+    return gson.toJson(this.reaper.collapse(this.worldUpdater));
+  }
+}

arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/AddressReaper.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package net.consensys.linea.blockcapture.reapers;
+
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.hyperledger.besu.datatypes.Address;
+
+public class AddressReaper {
+  private final ArrayDeque<Set<Address>> reaped = new ArrayDeque<>();
+
+  public AddressReaper() {
+    // “Bedrock” address set for block-level gathering.
+    this.reaped.addLast(new HashSet<>());
+  }
+
+  public void enterTransaction() {
+    this.reaped.addLast(new HashSet<>());
+  }
+
+  public void exitTransaction(boolean success) {
+    if (!success) {
+      this.reaped.removeLast();
+    }
+  }
+
+  public void touch(final Address... addresses) {
+    for (Address address : addresses) {
+      this.reaped.peekLast().add(address);
+    }
+  }
+
+  public Set<Address> collapse() {
+    return this.reaped.stream().flatMap(Collection::stream).collect(Collectors.toSet());
+  }
+}

arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/Reaper.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package net.consensys.linea.blockcapture.reapers;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import net.consensys.linea.blockcapture.snapshots.AccountSnapshot;
+import net.consensys.linea.blockcapture.snapshots.BlockSnapshot;
+import net.consensys.linea.blockcapture.snapshots.ConflationSnapshot;
+import net.consensys.linea.blockcapture.snapshots.StorageSnapshot;
+import org.apache.tuweni.units.bigints.UInt256;
+import org.hyperledger.besu.datatypes.Address;
+import org.hyperledger.besu.datatypes.Transaction;
+import org.hyperledger.besu.evm.worldstate.WorldUpdater;
+import org.hyperledger.besu.plugin.data.BlockBody;
+import org.hyperledger.besu.plugin.data.BlockHeader;
+
+/**
+ * The Reaper collect all the information from the state that will be accessed during the execution
+ * of a conflation.
+ *
+ * <p>This data can than be collapsed into a “replay” ({@link ConflationSnapshot}), i.e. the minimal
+ * required information to replay a conflation as if it were executed on the blockchain.
+ */
+public class Reaper {
+  /** Collect the reads from the state */
+  private final StorageReaper storage = new StorageReaper();
+  /** Collect the addresses read from the state */
+  private final AddressReaper addresses = new AddressReaper();
+  /** Collect the blocks within a conflation */
+  private final List<BlockSnapshot> blocks = new ArrayList<>();
+
+  public void enterBlock(final BlockHeader header, final BlockBody body) {
+    this.blocks.add(
+        BlockSnapshot.of((org.hyperledger.besu.ethereum.core.BlockHeader) header, body));
+    this.addresses.touch(header.getCoinbase());
+  }
+
+  public void enterTransaction(Transaction tx) {
+    this.storage.enterTransaction();
+    this.addresses.enterTransaction();
+
+    this.touchAddress(tx.getSender());
+    tx.getTo().ifPresent(this::touchAddress);
+  }
+
+  public void exitTransaction(boolean success) {
+    this.storage.exitTransaction(success);
+    this.addresses.exitTransaction(success);
+  }
+
+  public void touchAddress(final Address address) {
+    this.addresses.touch(address);
+  }
+
+  public void touchStorage(final Address address, final UInt256 key) {
+    this.storage.touch(address, key);
+  }
+
+  /**
+   * Uniquify and solidify the accumulated data, then return a {@link ConflationSnapshot}, which
+   * contains the smallest dataset required to exactly replay the conflation within a test framework
+   * without requiring access to the whole state.
+   *
+   * @param world the state before the conflation execution
+   * @return a minimal set of information required to replay the conflation within a test framework
+   */
+  public ConflationSnapshot collapse(final WorldUpdater world) {
+    final List<AccountSnapshot> initialAccounts =
+        this.addresses.collapse().stream()
+            .flatMap(a -> AccountSnapshot.from(a, world).stream())
+            .toList();
+
+    final List<StorageSnapshot> initialStorage = new ArrayList<>();
+    for (Map.Entry<Address, Set<UInt256>> e : this.storage.collapse().entrySet()) {
+      final Address address = e.getKey();
+
+      e.getValue().stream()
+          .flatMap(key -> StorageSnapshot.from(address, key, world).stream())
+          .forEach(initialStorage::add);
+    }
+
+    return new ConflationSnapshot(this.blocks, initialAccounts, initialStorage);
+  }
+}

arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/StorageReaper.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package net.consensys.linea.blockcapture.reapers;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.tuweni.units.bigints.UInt256;
+import org.hyperledger.besu.datatypes.Address;
+
+/**
+ * This object gathers all non-reversed accesses to storage values during the execution of a
+ * conflation, then collapse them into a single mapping of the initial values in these slots.
+ */
+public class StorageReaper {
+  private final Deque<HashMap<Address, Set<UInt256>>> transientStates = new ArrayDeque<>();
+
+  public void enterTransaction() {
+    this.transientStates.addLast(new HashMap<>());
+  }
+
+  public void exitTransaction(boolean success) {
+    if (!success) {
+      this.transientStates.removeLast();
+    }
+  }
+
+  public void touch(final Address address, final UInt256 key) {
+    this.transientStates.peekLast().computeIfAbsent(address, k -> new HashSet<>()).add(key);
+  }
+
+  public Map<Address, Set<UInt256>> collapse() {
+    final Map<Address, Set<UInt256>> r = new HashMap<>();
+
+    for (var txEntry : this.transientStates) {
+      for (Map.Entry<Address, Set<UInt256>> addressKeys : txEntry.entrySet()) {
+        final Address address = addressKeys.getKey();
+
+        // Use computeIfAbsent instead of put, as we only want to capture the **first** read.
+        r.computeIfAbsent(address, k -> new HashSet<>()).addAll(addressKeys.getValue());
+      }
+    }
+
+    return r;
+  }
+}

arithmetization/src/main/java/net/consensys/linea/blockcapture/snapshots/AccessListEntrySnapshot.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package net.consensys.linea.blockcapture.snapshots;
+
+import java.util.List;
+
+import org.hyperledger.besu.datatypes.AccessListEntry;
+
+public record AccessListEntrySnapshot(String address, List<String> storageKeys) {
+  public static AccessListEntrySnapshot from(AccessListEntry e) {
+    return new AccessListEntrySnapshot(e.getAddressString(), e.getStorageKeysString());
+  }
+}