Refactoring to use test watcher for unit tests as well.

arithmetization/build.gradle

@@ -71,6 +71,7 @@ dependencies {
   testImplementation "${besuArtifactGroup}.internal:rlp"
   testImplementation "${besuArtifactGroup}.internal:core"
   testImplementation "${besuArtifactGroup}.internal:referencetests"
+  testImplementation 'org.junit.platform:junit-platform-launcher'
 }
 
 configurations {

arithmetization/src/test/java/net/consensys/linea/UnitTestOutcomeWriter.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package net.consensys.linea;
+
+import static net.consensys.linea.reporting.TestOutcomeWriterTool.writeToJsonFile;
+
+import org.junit.platform.launcher.LauncherSession;
+import org.junit.platform.launcher.LauncherSessionListener;
+
+public class UnitTestOutcomeWriter implements LauncherSessionListener {
+
+  public static final String FILE_NAME = "UnitTestsResults.json";
+
+  @Override
+  public void launcherSessionClosed(LauncherSession session) {
+    writeToJsonFile(FILE_NAME);
+  }
+}

arithmetization/src/test/java/net/consensys/linea/UnitTestWatcher.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package net.consensys.linea;
+
+import java.util.Optional;
+
+import lombok.extern.slf4j.Slf4j;
+import net.consensys.linea.reporting.TestOutcomeWriterTool;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestWatcher;
+
+@Slf4j
+public class UnitTestWatcher implements TestWatcher {
+
+  private String FAILED = "FAILED";
+
+  @Override
+  public void testFailed(ExtensionContext context, Throwable cause) {
+    String testName = context.getDisplayName();
+    log.info("Adding failure for {}", testName);
+    TestOutcomeWriterTool.addFailure(
+        FAILED, cause.getMessage().split(System.lineSeparator(), 2)[0], testName);
+    log.info("Failure added for {}", testName);
+  }
+
+  @Override
+  public void testSuccessful(ExtensionContext context) {
+    TestOutcomeWriterTool.addSuccess();
+  }
+
+  @Override
+  public void testDisabled(ExtensionContext context, Optional<String> reason) {
+    TestOutcomeWriterTool.addSkipped();
+  }
+
+  @Override
+  public void testAborted(ExtensionContext context, Throwable cause) {
+    TestOutcomeWriterTool.addAborted();
+  }
+}

arithmetization/src/test/java/net/consensys/linea/zktracer/module/ecdata/ecpairing/EcPairingrTest.java

@@ -46,6 +46,7 @@
 // A TestWatcher is used to log the results of testEcPairingSingleForScenario
 // into a csv file (one for successful and one for failing cases)
 // that can be used to run the same test cases with @CsvFileSource
+
 @ExtendWith(EcPairingTestWatcher.class)
 public class EcPairingrTest {
   // https://github.com/Consensys/linea-arithmetization/issues/822

arithmetization/src/test/java/net/consensys/linea/zktracer/module/mxp/MxpTest.java

@@ -51,7 +51,6 @@
 import org.junit.jupiter.api.parallel.ExecutionMode;
 
 // https://github.com/Consensys/linea-besu-plugin/issues/197
-
 @Execution(ExecutionMode.SAME_THREAD)
 public class MxpTest {
 

reference-tests/build.gradle

@@ -123,6 +123,6 @@ dependencies {
 
   implementation project(":arithmetization")
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
-  testImplementation project(path: ':testing')
+  implementation project(path: ':testing')
   implementation 'org.junit.platform:junit-platform-launcher'
 }

reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java

@@ -14,7 +14,7 @@
  */
 package net.consensys.linea;
 
-import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.setFileDirectory;
+import static net.consensys.linea.reporting.TestOutcomeWriterTool.getFileDirectory;
 
 import java.io.IOException;
 import java.nio.file.FileAlreadyExistsException;
@@ -31,7 +31,7 @@ public class BlockchainReferenceTestJson {
 
   @Synchronized
   public static CompletableFuture<String> readBlockchainReferenceTestsOutput(String fileName) {
-    String fileDirectory = setFileDirectory();
+    String fileDirectory = getFileDirectory();
     return CompletableFuture.supplyAsync(
         () -> {
           Path directoryPath = Paths.get(fileDirectory);

reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java

@@ -14,24 +14,16 @@
  */
 package net.consensys.linea;
 
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import static net.consensys.linea.reporting.TestOutcomeWriterTool.addFailure;
+
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListSet;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.Synchronized;
 import lombok.extern.slf4j.Slf4j;
-import net.consensys.linea.zktracer.json.JsonConverter;
+import net.consensys.linea.reporting.TestOutcomeWriterTool;
+import net.consensys.linea.reporting.TestState;
 
 @Slf4j
 public class ReferenceTestOutcomeRecorderTool {
@@ -42,51 +34,23 @@ public class ReferenceTestOutcomeRecorderTool {
   public static final String JSON_OUTPUT_FILENAME =
       System.getenv()
           .getOrDefault("REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "failedReferenceTests.json");
-  public static JsonConverter jsonConverter = JsonConverter.builder().build();
-  private static volatile AtomicInteger failedCounter = new AtomicInteger(0);
-  private static volatile AtomicInteger successCounter = new AtomicInteger(0);
-  private static volatile AtomicInteger disabledCounter = new AtomicInteger(0);
-  private static volatile AtomicInteger abortedCounter = new AtomicInteger(0);
-  private static volatile ConcurrentMap<
-          String, ConcurrentMap<String, ConcurrentSkipListSet<String>>>
-      modulesToConstraintsToTests = new ConcurrentHashMap<>();
 
   public static void mapAndStoreTestResult(
       String testName, TestState success, Map<String, Set<String>> failedConstraints) {
     switch (success) {
       case FAILED -> {
-        failedCounter.incrementAndGet();
+        TestOutcomeWriterTool.addFailed();
         for (Map.Entry<String, Set<String>> failedConstraint : failedConstraints.entrySet()) {
           String moduleName = failedConstraint.getKey();
           for (String constraint : failedConstraint.getValue()) {
-            ConcurrentMap<String, ConcurrentSkipListSet<String>> constraintsToTests =
-                modulesToConstraintsToTests.computeIfAbsent(
-                    moduleName, m -> new ConcurrentHashMap<>());
-            ConcurrentSkipListSet<String> failingTests =
-                constraintsToTests.computeIfAbsent(constraint, m -> new ConcurrentSkipListSet<>());
-            int size = failingTests.size();
-            failingTests.add(testName);
-            if (failingTests.size() == size) {
-              log.warn("Duplicate name found... {}", failedConstraint);
-            }
+            addFailure(moduleName, constraint, testName);
           }
         }
       }
-      case SUCCESS -> successCounter.incrementAndGet();
-      case ABORTED -> abortedCounter.incrementAndGet();
-      case DISABLED -> disabledCounter.incrementAndGet();
-    }
-  }
-
-  @Synchronized
-  public static BlockchainReferenceTestOutcome parseBlockchainReferenceTestOutcome(
-      String jsonString) {
-    if (!jsonString.isEmpty()) {
-      BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
-          jsonConverter.fromJson(jsonString, BlockchainReferenceTestOutcome.class);
-      return blockchainReferenceTestOutcome;
+      case SUCCESS -> TestOutcomeWriterTool.addSuccess();
+      case ABORTED -> TestOutcomeWriterTool.addAborted();
+      case DISABLED -> TestOutcomeWriterTool.addSkipped();
     }
-    throw new RuntimeException("invalid JSON");
   }
 
   public static Map<String, Set<String>> extractConstraints(String message) {
@@ -174,57 +138,5 @@ private static void getPairFromString(String constraint, Map<String, Set<String>
     pairs.computeIfAbsent(pair[0].trim(), p -> new HashSet<>()).add(pair[1].trim());
   }
 
-  @Synchronized
-  public static void writeToJsonFile() {
-    try {
-      String directory = setFileDirectory();
-      log.info("Reference test will be written to file {} \\ {}", directory, JSON_OUTPUT_FILENAME);
-      writeToJsonFileInternal(JSON_OUTPUT_FILENAME).get();
-      log.info("Reference test results written to file {}", JSON_OUTPUT_FILENAME);
-      log.info(
-          "Path exists: {}, file exist: {}",
-          Paths.get(directory).toFile().exists(),
-          Paths.get(directory).resolve(JSON_OUTPUT_FILENAME).toFile().exists());
-    } catch (Exception e) {
-      log.error("Error while writing results");
-      throw new RuntimeException("Error while writing results", e);
-    }
-  }
-
   static ObjectMapper objectMapper = new ObjectMapper();
-
-  @Synchronized
-  private static CompletableFuture<Void> writeToJsonFileInternal(String name) {
-    String fileDirectory = setFileDirectory();
-    log.info("writing results summary to {}", fileDirectory + "/" + name);
-    try {
-      Files.createDirectories(Path.of(fileDirectory));
-    } catch (IOException e) {
-      log.error("Error - Failed to create test directory output: %s".formatted(e.getMessage()));
-      throw new RuntimeException(e);
-    }
-    return CompletableFuture.runAsync(
-        () -> {
-          try (FileWriter file = new FileWriter(Path.of(fileDirectory, name).toString())) {
-            objectMapper.writeValue(
-                file,
-                new BlockchainReferenceTestOutcome(
-                    failedCounter.get(),
-                    successCounter.get(),
-                    disabledCounter.get(),
-                    abortedCounter.get(),
-                    modulesToConstraintsToTests));
-          } catch (Exception e) {
-            log.error("Error - Failed to write test output: %s".formatted(e.getMessage()));
-          }
-        });
-  }
-
-  static String setFileDirectory() {
-    String jsonDirectory = System.getenv("FAILED_TEST_JSON_DIRECTORY");
-    if (jsonDirectory == null || jsonDirectory.isEmpty()) {
-      return "../tmp/local/";
-    }
-    return jsonDirectory;
-  }
 }

reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeWriter.java

@@ -17,6 +17,7 @@
 import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.*;
 
 import lombok.extern.slf4j.Slf4j;
+import net.consensys.linea.reporting.TestOutcomeWriterTool;
 import org.junit.platform.launcher.LauncherSession;
 import org.junit.platform.launcher.LauncherSessionListener;
 
@@ -25,6 +26,6 @@ public class ReferenceTestOutcomeWriter implements LauncherSessionListener {
 
   @Override
   public void launcherSessionClosed(LauncherSession session) {
-    writeToJsonFile();
+    TestOutcomeWriterTool.writeToJsonFile(JSON_OUTPUT_FILENAME);
   }
 }

reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java

@@ -36,6 +36,8 @@
 
 import lombok.extern.slf4j.Slf4j;
 import net.consensys.linea.corset.CorsetValidator;
+import net.consensys.linea.reporting.TestOutcome;
+import net.consensys.linea.reporting.TestOutcomeWriterTool;
 import net.consensys.linea.testing.ExecutionEnvironment;
 import net.consensys.linea.zktracer.ZkTracer;
 import org.hyperledger.besu.ethereum.MainnetBlockValidator;
@@ -94,6 +96,18 @@ public class BlockchainReferenceTestTools {
     PARAMS.ignore("RevertInCreateInInitCreate2_d0g0v0_London[London]");
     PARAMS.ignore("RevertInCreateInInit_d0g0v0_London[London]");
 
+    // Arithmetization restriction: recipient address is a precompile.
+    PARAMS.ignore("modexpRandomInput_d0g0v0_London[London]");
+    PARAMS.ignore("modexpRandomInput_d0g1v0_London[London]");
+    PARAMS.ignore("modexpRandomInput_d1g0v0_London[London]");
+    PARAMS.ignore("modexpRandomInput_d1g1v0_London[London]");
+    PARAMS.ignore("modexpRandomInput_d2g0v0_London[London]");
+    PARAMS.ignore("modexpRandomInput_d2g1v0_London[London]");
+    PARAMS.ignore("randomStatetest642_d0g0v0_London[London]");
+    PARAMS.ignore("randomStatetest644_d0g0v0_London[London]");
+    PARAMS.ignore("randomStatetest645_d0g0v0_London[London]");
+    PARAMS.ignore("randomStatetest645_d0g0v1_London[London]");
+
     // Consumes a huge amount of memory.
     PARAMS.ignore("static_Call1MB1024Calldepth_d1g0v0_\\w+");
     PARAMS.ignore("ShanghaiLove_.*");
@@ -105,6 +119,16 @@ public class BlockchainReferenceTestTools {
     // Absurd amount of gas, doesn't run in parallel.
     PARAMS.ignore("randomStatetest94_\\w+");
 
+    // Balance is more than 128 bits
+    PARAMS.ignore("Call1024PreCalls_d0g0v0_London[London]");
+    PARAMS.ignore("Call1024PreCalls_d0g1v0_London[London]");
+    PARAMS.ignore("OverflowGasRequire_London[London]");
+    PARAMS.ignore("StrangeContractCreation_London[London]");
+    PARAMS.ignore("SuicideIssue_London[London]");
+    PARAMS.ignore("DelegateCallSpam_London[London]");
+    PARAMS.ignore("OverflowGasRequire2_d0g0v0_London[London]");
+    PARAMS.ignore("HighGasLimit_d0g0v0_London[London]");
+
     // Don't do time-consuming tests.
     PARAMS.ignore("CALLBlake2f_MaxRounds.*");
     PARAMS.ignore("loopMul_*");
@@ -129,9 +153,9 @@ public static CompletableFuture<Set<String>> getRecordedFailedTestsFromJson(
       return CompletableFuture.completedFuture(failedTests);
     }
 
-    CompletableFuture<BlockchainReferenceTestOutcome> modulesToConstraintsFutures =
+    CompletableFuture<TestOutcome> modulesToConstraintsFutures =
         readBlockchainReferenceTestsOutput(JSON_INPUT_FILENAME)
-            .thenApply(ReferenceTestOutcomeRecorderTool::parseBlockchainReferenceTestOutcome);
+            .thenApply(TestOutcomeWriterTool::parseTestOutcome);
 
     return modulesToConstraintsFutures.thenApply(
         blockchainReferenceTestOutcome -> {

reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java

@@ -25,6 +25,9 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListSet;
 
+import net.consensys.linea.reporting.TestOutcome;
+import net.consensys.linea.reporting.TestOutcomeWriterTool;
+import net.consensys.linea.reporting.TestState;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -52,13 +55,13 @@ void multipleModulesAreStoredCorrectly() {
         "test1", TestState.FAILED, Map.of("Constraint", Set.of(module1, module2)));
     ReferenceTestOutcomeRecorderTool.mapAndStoreTestResult(
         "test2", TestState.FAILED, Map.of("Constraint", Set.of(module1)));
-    ReferenceTestOutcomeRecorderTool.writeToJsonFile();
+    TestOutcomeWriterTool.writeToJsonFile(JSON_OUTPUT_FILENAME_TEST);
 
     readBlockchainReferenceTestsOutput(JSON_OUTPUT_FILENAME_TEST)
         .thenApply(
             jsonString -> {
-              BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
-                  ReferenceTestOutcomeRecorderTool.parseBlockchainReferenceTestOutcome(jsonString);
+              TestOutcome blockchainReferenceTestOutcome =
+                  TestOutcomeWriterTool.parseTestOutcome(jsonString);
 
               ConcurrentMap<String, ConcurrentMap<String, ConcurrentSkipListSet<String>>>
                   modulesToConstraints =
@@ -155,8 +158,8 @@ public void extractConstraints() {
 
   @Test
   void parseBlockchainReferenceTestOutcome() {
-    BlockchainReferenceTestOutcome outcome =
-        ReferenceTestOutcomeRecorderTool.parseBlockchainReferenceTestOutcome(
+    TestOutcome outcome =
+        TestOutcomeWriterTool.parseTestOutcome(
             """
             {
               "abortedCounter": 20,

reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java

@@ -14,7 +14,7 @@
  */
 package net.consensys.linea;
 
-import static net.consensys.linea.TestState.*;
+import static net.consensys.linea.reporting.TestState.*;
 import static net.consensys.linea.testing.ExecutionEnvironment.CORSET_VALIDATION_RESULT;
 
 import java.util.HashMap;