Merge branch 'arith-dev' into perf/issue-716/improve-performance-ZkTr…

…acer-contructor

.github/workflows/gradle.yml

@@ -22,11 +22,11 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v3
         with:
-          java-version: '17'
-          distribution: 'adopt'
+          java-version: 21
+          distribution: temurin
 
       - name: Build without tests
         run: ./gradlew build -x test -x spotlessCheck
@@ -52,11 +52,14 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v3
         with:
-          java-version: '17'
-          distribution: 'adopt'
+          java-version: 21
+          distribution: temurin
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
 
       # The asset URL for the latest release can be found with:
       # curl -L -H "Accept: application/vnd.github+json"  \
@@ -65,21 +68,23 @@ jobs:
       #     https://api.github.com/repos/ConsenSys/corset/releases/latest
       # | jq '.assets[] | select(.name|endswith("x86_64-unknown-linux-musl.tar.gz")) | .url'
       - name: Install Corset
-        run: |
-          curl -L \
-          -H "Accept: application/octet-stream" \
-          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-          -H "X-GitHub-Api-Version: 2022-11-28" \
-          -o corset.tar.gz \
-          https://api.github.com/repos/Consensys/corset/releases/assets/151396061
-          tar xzf corset.tar.gz
-          mv corset $HOME
-          echo $HOME >> $GITHUB_PATH
+        run: RUSTFLAGS=-Awarnings cargo install --git ssh://git@github.com/ConsenSys/corset --tag v9.7.11 --locked --force
+#        run: |
+#          curl -L \
+#          -H "Accept: application/octet-stream" \
+#          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+#          -H "X-GitHub-Api-Version: 2022-11-28" \
+#          -o corset.tar.gz \
+#          https://api.github.com/repos/Consensys/corset/releases/assets/151396061
+#          tar xzf corset.tar.gz
+#          mv corset $HOME
+#          echo $HOME >> $GITHUB_PATH
 
       - name: Run unit tests
         run: ./gradlew :arithmetization:test
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
+          CORSET_FLAGS: fields,expand,expand,expand
 
       - name: Upload test report
         if: always()
@@ -96,7 +101,7 @@ jobs:
       - name: Set up Java
         uses: actions/setup-java@v3
         with:
-          distribution: adopt
-          java-version: 17
+          distribution: temurin
+          java-version: 21
       - name: spotless
         run: ./gradlew --no-daemon --parallel clean spotlessCheck

.github/workflows/release.yml

@@ -11,11 +11,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v3
         with:
-          java-version: '17'
-          distribution: 'adopt'
+          java-version: 21
+          distribution: temurin
           cache: 'gradle'
 
       - name: Grant execute permission for gradlew
@@ -65,11 +65,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v3
         with:
-          java-version: 17
-          distribution: 'adopt'
+          java-version: 21
+          distribution: temurin
 
       - name: Publish Java artifacts
         run: ./gradlew publish

CHANGELOG.md

@@ -1,10 +1,23 @@
 # Changelog
 
+## 0.2.0-rc1
+* feat: add PRECOMPILE_ECPAIRING_G2_MEMBERSHIP_CALLS in spillings.toml and did some renaming ([#819](https://github.com/Consensys/linea-arithmetization/pull/819))
+* feat: optimise trace generation (except hub) ([#838](https://github.com/Consensys/linea-arithmetization/pull/838))
+
+## 0.1.5-rc6
+* Migrating of `TRACES_DIR` env var to 
+`plugin-linea-conflated-trace-generation-traces-output-path` CLI option that can be included in the toml config files.
+The path specified in `plugin-linea-conflated-trace-generation-traces-output-path` will be created automatically if it does not exist.
+This time this has nothing to do with the `ContinuousTracingPlugin` [#830](https://github.com/Consensys/linea-arithmetization/pull/830).
+
+## 0.1.5-rc5
+* Migrating of `TRACES_DIR` env var to `plugin-linea-continuous-tracing-traces-dir` CLI option that can be included in the 
+toml config files. The path specified in `plugin-linea-continuous-tracing-traces-dir` will be created automatically 
+  if it does not exist [#825](https://github.com/Consensys/linea-arithmetization/pull/825).
+
 ## 0.1.4-test21
-Test pre-release 21 from [temp/issue-248/count-stack-only](https://github.
-com/Consensys/linea-arithmetization/tree/temp/issue-248/count-stack-only)
-* fix: capture SSTORE-touched storage slots for correct gas computations [#606](https://github.
-  com/Consensys/linea-arithmetization/pull/606)
+Test pre-release 21 from [temp/issue-248/count-stack-only](https://github.com/Consensys/linea-arithmetization/tree/temp/issue-248/count-stack-only)
+* fix: capture SSTORE-touched storage slots for correct gas computations [#606](https://github.com/Consensys/linea-arithmetization/pull/606)
 * build: make the build script portable, explicit dependency on Go & GCC, test libcompress build [#621](https://github.com/Consensys/linea-arithmetization/pull/621)
 * Update after the refactor of transaction selection service [#626](https://github.com/Consensys/linea-arithmetization/pull/626)
 * Use the right classloader to load the native library [#628](https://github.com/Consensys/linea-arithmetization/pull/628)

README.md

@@ -5,10 +5,10 @@ an [existing implementation in Go](https://github.com/Consensys/zk-evm/).
 
 ## Development Setup
 
-### Install Java 17
+### Install Java 21
 
 ```
-brew install openjdk@17
+brew install openjdk@21
 ```
 
 ### Install the Go toolchain
@@ -155,6 +155,19 @@ ______________________________________________________________________
 corset check -T <JSON_FILE> -v zkevm-constraints/zkevm.bin
 ```
 
+## Disable Corset Expansion
+
+Corset expansion means that generated traces are checked as accurately
+as possible.  However, this slows testing down to some extent.  It can
+be easily disabled in IntelliJ:
+
+   - Go to `Run | Edit Configurations`
+
+   ![idea_disable_corset_expansion.png](images/idea_disable_corset_expansion.png)
+
+   and add `CORSET_FLAGS=` under `Environment Variables`.  This turns
+   off all expansion modes, including field arithmetic.
+
 ## Plugins
 
 Plugins are documented [here](PLUGINS.md).

arithmetization/build.gradle

@@ -58,7 +58,6 @@ dependencies {
   implementation 'io.tmio:tuweni-bytes'
   implementation 'io.tmio:tuweni-units'
   implementation 'io.tmio:tuweni-toml'
-
   implementation 'org.bouncycastle:bcprov-jdk18on'
   implementation 'org.hibernate.validator:hibernate-validator'
 

arithmetization/src/main/java/net/consensys/linea/blockcapture/BlockCapturer.java

@@ -16,11 +16,13 @@
 package net.consensys.linea.blockcapture;
 
 import java.util.List;
+import java.util.Set;
 
 import com.google.gson.Gson;
 import net.consensys.linea.blockcapture.reapers.Reaper;
 import net.consensys.linea.zktracer.ConflationAwareOperationTracer;
 import net.consensys.linea.zktracer.opcode.OpCode;
+import net.consensys.linea.zktracer.types.AddressUtils;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.units.bigints.UInt256;
 import org.hyperledger.besu.datatypes.Address;
@@ -39,6 +41,7 @@ public class BlockCapturer implements ConflationAwareOperationTracer {
    * The {@link Reaper} will collect all the data that will need to be mimicked to replay the block.
    */
   private final Reaper reaper = new Reaper();
+
   /**
    * This keeps a pointer to the initial state (i.e. ) to be used at the end of tracing to store the
    * minimal required information to replay the conflation.
@@ -78,6 +81,7 @@ public void traceEndTransaction(
       Bytes output,
       List<Log> logs,
       long gasUsed,
+      Set<Address> selfDestructs,
       long timeNs) {
     this.reaper.exitTransaction(status);
   }
@@ -128,6 +132,21 @@ public void tracePreExecution(MessageFrame frame) {
           this.reaper.touchAddress(target);
         }
       }
+
+      case BALANCE -> {
+        if (frame.stackSize() > 0) {
+          final Address target = Words.toAddress(frame.getStackItem(0));
+          this.reaper.touchAddress(target);
+        }
+      }
+
+        // Failure condition if created address already exists
+      case CREATE, CREATE2 -> {
+        if (frame.stackSize() > 0) {
+          final Address target = AddressUtils.getDeploymentAddress(frame);
+          this.reaper.touchAddress(target);
+        }
+      }
     }
   }
 

arithmetization/src/main/java/net/consensys/linea/blockcapture/reapers/Reaper.java

@@ -41,8 +41,10 @@
 public class Reaper {
   /** Collect the reads from the state */
   private final StorageReaper storage = new StorageReaper();
+
   /** Collect the addresses read from the state */
   private final AddressReaper addresses = new AddressReaper();
+
   /** Collect the blocks within a conflation */
   private final List<BlockSnapshot> blocks = new ArrayList<>();