Blockchain reference tests report - success and failure counters

reference-tests/src/main/java/net/consensys/linea/FailedTestJson.java → reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java

@@ -26,11 +26,11 @@
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-public class FailedTestJson {
+public class BlockchainReferenceTestJson {
   static String fileDirectory = setFileDirectory();
 
   @Synchronized
-  public static CompletableFuture<String> readFailedTestsOutput(String fileName) {
+  public static CompletableFuture<String> readBlockchainReferenceTestsOutput(String fileName) {
     return CompletableFuture.supplyAsync(
         () -> {
           Path directoryPath = Paths.get(fileDirectory);

reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package net.consensys.linea;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+@JsonPropertyOrder({"failedCounter", "successCounter", "modules"})
+public record BlockchainReferenceTestOutcome(
+    int failedCounter, int successCounter, List<ModuleToConstraints> modulesToConstraints) {}

reference-tests/src/main/java/net/consensys/linea/ModuleToConstraints.java

@@ -21,17 +21,9 @@
 import java.util.stream.Collectors;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
 public record ModuleToConstraints(String moduleName, Map<String, Set<String>> constraints) {
 
-  public ModuleToConstraints(
-      @JsonProperty("moduleName") String moduleName,
-      @JsonProperty("constraints") Map<String, Set<String>> constraints) {
-    this.moduleName = moduleName;
-    this.constraints = constraints;
-  }
-
   @Override
   public boolean equals(Object o) {
     if (this == o) {

reference-tests/src/main/java/net/consensys/linea/MapFailedReferenceTestsTool.java → reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java

@@ -14,8 +14,8 @@
  */
 package net.consensys.linea;
 
-import static net.consensys.linea.FailedTestJson.readFailedTestsOutput;
-import static net.consensys.linea.FailedTestJson.writeToJsonFile;
+import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput;
+import static net.consensys.linea.BlockchainReferenceTestJson.writeToJsonFile;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -33,31 +33,58 @@
 import net.consensys.linea.zktracer.json.JsonConverter;
 
 @Slf4j
-public class MapFailedReferenceTestsTool {
+public class ReferenceTestOutcomeRecorderTool {
+
+  static JsonConverter jsonConverter = JsonConverter.builder().build();
 
   @Synchronized
   public static void mapAndStoreFailedReferenceTest(
       String testName, List<String> logEventMessages, String jsonOutputFilename) {
     Set<String> failedConstraints = getFailedConstraints(logEventMessages);
     if (!failedConstraints.isEmpty()) {
-      readFailedTestsOutput(jsonOutputFilename)
+      readBlockchainReferenceTestsOutput(jsonOutputFilename)
           .thenCompose(
               jsonString -> {
                 JsonConverter jsonConverter = JsonConverter.builder().build();
 
-                List<ModuleToConstraints> modulesToConstraints =
-                    getModulesToConstraints(jsonString, jsonConverter);
+                BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
+                    getBlockchainReferenceTestOutcome(jsonString);
 
                 mapFailedConstraintsToTestsToModule(
-                    modulesToConstraints, failedConstraints, testName);
-
-                jsonString = jsonConverter.toJson(modulesToConstraints);
+                    blockchainReferenceTestOutcome.modulesToConstraints(),
+                    failedConstraints,
+                    testName);
+
+                BlockchainReferenceTestOutcome finalBlockchainReferenceTestOutcome =
+                    new BlockchainReferenceTestOutcome(
+                        blockchainReferenceTestOutcome.failedCounter() + 1,
+                        blockchainReferenceTestOutcome.successCounter(),
+                        blockchainReferenceTestOutcome.modulesToConstraints());
+                jsonString = jsonConverter.toJson(finalBlockchainReferenceTestOutcome);
                 writeToJsonFile(jsonString, jsonOutputFilename);
                 return null;
               });
     }
   }
 
+  @Synchronized
+  public static void incrementSuccessRate(String jsonOutputFilename) {
+    readBlockchainReferenceTestsOutput(jsonOutputFilename)
+        .thenAccept(
+            jsonString -> {
+              BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
+                  getBlockchainReferenceTestOutcome(jsonString);
+
+              BlockchainReferenceTestOutcome finalBlockchainReferenceTestOutcome =
+                  new BlockchainReferenceTestOutcome(
+                      blockchainReferenceTestOutcome.failedCounter(),
+                      blockchainReferenceTestOutcome.successCounter() + 1,
+                      blockchainReferenceTestOutcome.modulesToConstraints());
+              jsonString = jsonConverter.toJson(finalBlockchainReferenceTestOutcome);
+              writeToJsonFile(jsonString, jsonOutputFilename);
+            });
+  }
+
   @Synchronized
   private static void mapFailedConstraintsToTestsToModule(
       List<ModuleToConstraints> modulesToConstraints,
@@ -119,15 +146,16 @@ private static void addModuleIfAbsent(
   }
 
   @Synchronized
-  public static List<ModuleToConstraints> getModulesToConstraints(
-      String jsonString, JsonConverter jsonConverter) {
-    List<ModuleToConstraints> moduleToConstraints = new ArrayList<>();
+  public static BlockchainReferenceTestOutcome getBlockchainReferenceTestOutcome(
+      String jsonString) {
+    BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = null;
     if (!jsonString.isEmpty()) {
-      moduleToConstraints =
-          new ArrayList<>(
-              Arrays.asList(jsonConverter.fromJson(jsonString, ModuleToConstraints[].class)));
+      blockchainReferenceTestOutcome =
+          jsonConverter.fromJson(jsonString, BlockchainReferenceTestOutcome.class);
+    } else {
+      blockchainReferenceTestOutcome = new BlockchainReferenceTestOutcome(0, 0, new ArrayList<>());
     }
-    return moduleToConstraints;
+    return blockchainReferenceTestOutcome;
   }
 
   private static Set<String> getFailedConstraints(List<String> logEventMessages) {

reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java

@@ -15,9 +15,8 @@
 
 package net.consensys.linea;
 
-import static net.consensys.linea.FailedTestJson.readFailedTestsOutput;
-import static net.consensys.linea.MapFailedReferenceTestsTool.getModule;
-import static net.consensys.linea.MapFailedReferenceTestsTool.getModulesToConstraints;
+import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput;
+import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.getModule;
 import static net.consensys.linea.ReferenceTestWatcher.JSON_INPUT_FILENAME;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -36,7 +35,6 @@
 import net.consensys.linea.corset.CorsetValidator;
 import net.consensys.linea.testing.ExecutionEnvironment;
 import net.consensys.linea.zktracer.ZkTracer;
-import net.consensys.linea.zktracer.json.JsonConverter;
 import org.hyperledger.besu.ethereum.MainnetBlockValidator;
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
@@ -107,14 +105,15 @@ public static CompletableFuture<Set<String>> getRecordedFailedTestsFromJson(
     if (failedModule.isEmpty()) {
       return CompletableFuture.completedFuture(failedTests);
     }
-    JsonConverter jsonConverter = JsonConverter.builder().build();
-    CompletableFuture<List<ModuleToConstraints>> modulesToConstraintsFutures =
-        readFailedTestsOutput(JSON_INPUT_FILENAME)
-            .thenApply(jsonString -> getModulesToConstraints(jsonString, jsonConverter));
+
+    CompletableFuture<BlockchainReferenceTestOutcome> modulesToConstraintsFutures =
+        readBlockchainReferenceTestsOutput(JSON_INPUT_FILENAME)
+            .thenApply(ReferenceTestOutcomeRecorderTool::getBlockchainReferenceTestOutcome);
 
     return modulesToConstraintsFutures.thenApply(
-        modulesToConstraints -> {
-          ModuleToConstraints filteredFailedTests = getModule(modulesToConstraints, failedModule);
+        blockchainReferenceTestOutcome -> {
+          ModuleToConstraints filteredFailedTests =
+              getModule(blockchainReferenceTestOutcome.modulesToConstraints(), failedModule);
           if (filteredFailedTests == null) {
             return failedTests;
           }

reference-tests/src/test/java/net/consensys/linea/MapFailedReferenceTestsToolTest.java → reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java

@@ -14,9 +14,9 @@
  */
 package net.consensys.linea;
 
-import static net.consensys.linea.FailedTestJson.readFailedTestsOutput;
-import static net.consensys.linea.MapFailedReferenceTestsTool.getModulesToConstraints;
-import static net.consensys.linea.MapFailedReferenceTestsTool.mapAndStoreFailedReferenceTest;
+import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput;
+import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.getBlockchainReferenceTestOutcome;
+import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.mapAndStoreFailedReferenceTest;
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.io.File;
@@ -25,14 +25,12 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import net.consensys.linea.zktracer.json.JsonConverter;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-public class MapFailedReferenceTestsToolTest {
+public class ReferenceTestOutcomeRecorderToolTest {
 
   public static final String TEST_OUTPUT_JSON = "MapFailedReferenceTestsToolTest.json";
-  JsonConverter jsonConverter = JsonConverter.builder().build();
 
   @BeforeEach
   void setup() {
@@ -59,11 +57,14 @@ void multipleModulesAreStoredCorrectly() {
 
     mapAndStoreFailedReferenceTest(testName, List.of(dummyEvent), TEST_OUTPUT_JSON);
 
-    readFailedTestsOutput(TEST_OUTPUT_JSON)
+    readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON)
         .thenApply(
             jsonString -> {
+              BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
+                  getBlockchainReferenceTestOutcome(jsonString);
+
               List<ModuleToConstraints> modulesToConstraints =
-                  getModulesToConstraints(jsonString, jsonConverter);
+                  blockchainReferenceTestOutcome.modulesToConstraints();
 
               assertThat(modulesToConstraints.size()).isEqualTo(modules.size());
               assertThat(modulesToConstraints.get(0).moduleName()).isEqualTo(module1);
@@ -87,11 +88,14 @@ void multipleConstraintsInSameModuleAreStoredCorrectly() {
 
     mapAndStoreFailedReferenceTest(testName, List.of(dummyEvent), TEST_OUTPUT_JSON);
 
-    readFailedTestsOutput(TEST_OUTPUT_JSON)
+    readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON)
         .thenCompose(
             jsonString -> {
+              BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
+                  getBlockchainReferenceTestOutcome(jsonString);
+
               List<ModuleToConstraints> modulesToConstraints =
-                  getModulesToConstraints(jsonString, jsonConverter);
+                  blockchainReferenceTestOutcome.modulesToConstraints();
 
               assertThat(modulesToConstraints.size()).isEqualTo(1);
 
@@ -125,11 +129,14 @@ void multipleFailedTestsWithSameConstraintAreStoredCorrectly() {
     mapAndStoreFailedReferenceTest(testName3, List.of(dummyEvent), TEST_OUTPUT_JSON);
     mapAndStoreFailedReferenceTest(testName3, List.of(dummyEvent), TEST_OUTPUT_JSON);
 
-    readFailedTestsOutput(TEST_OUTPUT_JSON)
+    readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON)
         .thenCompose(
             jsonString -> {
+              BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
+                  getBlockchainReferenceTestOutcome(jsonString);
+
               List<ModuleToConstraints> modulesToConstraints =
-                  getModulesToConstraints(jsonString, jsonConverter);
+                  blockchainReferenceTestOutcome.modulesToConstraints();
 
               assertThat(modulesToConstraints.size()).isEqualTo(modules.size());
               Set<String> failedTests =

reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java

@@ -14,6 +14,7 @@
  */
 package net.consensys.linea;
 
+import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.incrementSuccessRate;
 import static org.junit.jupiter.api.Assumptions.abort;
 
 import java.time.Duration;
@@ -49,10 +50,15 @@ public void testFailed(ExtensionContext context, Throwable cause) {
     List<String> logEventMessages =
         listAppender.list.stream().map(ILoggingEvent::getMessage).toList();
 
-    MapFailedReferenceTestsTool.mapAndStoreFailedReferenceTest(
+    ReferenceTestOutcomeRecorderTool.mapAndStoreFailedReferenceTest(
         testName, logEventMessages, JSON_OUTPUT_FILENAME);
   }
 
+  @Override
+  public void testSuccessful(ExtensionContext context) {
+    incrementSuccessRate(JSON_OUTPUT_FILENAME);
+  }
+
   @Synchronized
   private static Logger getLogbackLogger() {
     try {