diff --git a/arithmetization/build.gradle b/arithmetization/build.gradle index fa063329c8..980f759d41 100644 --- a/arithmetization/build.gradle +++ b/arithmetization/build.gradle @@ -71,6 +71,7 @@ dependencies { testImplementation "${besuArtifactGroup}.internal:rlp" testImplementation "${besuArtifactGroup}.internal:core" testImplementation "${besuArtifactGroup}.internal:referencetests" + testImplementation 'org.junit.platform:junit-platform-launcher' } configurations { diff --git a/arithmetization/src/test/java/net/consensys/linea/UnitTestOutcomeWriter.java b/arithmetization/src/test/java/net/consensys/linea/UnitTestOutcomeWriter.java new file mode 100644 index 0000000000..6b0050d900 --- /dev/null +++ b/arithmetization/src/test/java/net/consensys/linea/UnitTestOutcomeWriter.java @@ -0,0 +1,30 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea; + +import static net.consensys.linea.reporting.TestOutcomeWriterTool.writeToJsonFile; + +import org.junit.platform.launcher.LauncherSession; +import org.junit.platform.launcher.LauncherSessionListener; + +public class UnitTestOutcomeWriter implements LauncherSessionListener { + + public static final String FILE_NAME = "UnitTestsResults.json"; + + @Override + public void launcherSessionClosed(LauncherSession session) { + writeToJsonFile(FILE_NAME); + } +} diff --git a/arithmetization/src/test/java/net/consensys/linea/UnitTestWatcher.java b/arithmetization/src/test/java/net/consensys/linea/UnitTestWatcher.java new file mode 100644 index 0000000000..b12fbcd54e --- /dev/null +++ b/arithmetization/src/test/java/net/consensys/linea/UnitTestWatcher.java @@ -0,0 +1,52 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea; + +import java.util.Optional; + +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.reporting.TestOutcomeWriterTool; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TestWatcher; + +@Slf4j +public class UnitTestWatcher implements TestWatcher { + + private String FAILED = "FAILED"; + + @Override + public void testFailed(ExtensionContext context, Throwable cause) { + String testName = context.getDisplayName(); + log.info("Adding failure for {}", testName); + TestOutcomeWriterTool.addFailure( + FAILED, cause.getMessage().split(System.lineSeparator(), 2)[0], testName); + log.info("Failure added for {}", testName); + } + + @Override + public void testSuccessful(ExtensionContext context) { + TestOutcomeWriterTool.addSuccess(); + } + + @Override + public void testDisabled(ExtensionContext context, Optional reason) { + TestOutcomeWriterTool.addSkipped(); + } + + @Override + public void testAborted(ExtensionContext context, Throwable cause) { + TestOutcomeWriterTool.addAborted(); + } +} diff --git a/arithmetization/src/test/java/net/consensys/linea/zktracer/module/ecdata/ecpairing/EcPairingrTest.java b/arithmetization/src/test/java/net/consensys/linea/zktracer/module/ecdata/ecpairing/EcPairingrTest.java index b85bdad399..aa9f533758 100644 --- a/arithmetization/src/test/java/net/consensys/linea/zktracer/module/ecdata/ecpairing/EcPairingrTest.java +++ b/arithmetization/src/test/java/net/consensys/linea/zktracer/module/ecdata/ecpairing/EcPairingrTest.java @@ -46,6 +46,7 @@ // A TestWatcher is used to log the results of testEcPairingSingleForScenario // into a csv file (one for successful and one for failing cases) // that can be used to run the same test cases with @CsvFileSource + @ExtendWith(EcPairingTestWatcher.class) public class EcPairingrTest { // https://github.com/Consensys/linea-arithmetization/issues/822 diff --git a/arithmetization/src/test/java/net/consensys/linea/zktracer/module/mxp/MxpTest.java b/arithmetization/src/test/java/net/consensys/linea/zktracer/module/mxp/MxpTest.java index 0461998fcb..94402c5744 100644 --- a/arithmetization/src/test/java/net/consensys/linea/zktracer/module/mxp/MxpTest.java +++ b/arithmetization/src/test/java/net/consensys/linea/zktracer/module/mxp/MxpTest.java @@ -51,7 +51,6 @@ import org.junit.jupiter.api.parallel.ExecutionMode; // https://github.com/Consensys/linea-besu-plugin/issues/197 - @Execution(ExecutionMode.SAME_THREAD) public class MxpTest { diff --git a/reference-tests/build.gradle b/reference-tests/build.gradle index 6b7722ce4f..715878b6a7 100644 --- a/reference-tests/build.gradle +++ b/reference-tests/build.gradle @@ -123,6 +123,6 @@ dependencies { implementation project(":arithmetization") implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - testImplementation project(path: ':testing') + implementation project(path: ':testing') implementation 'org.junit.platform:junit-platform-launcher' } diff --git a/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java b/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java index d362acc6be..081dc4853e 100644 --- a/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java +++ b/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java @@ -14,7 +14,7 @@ */ package net.consensys.linea; -import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.setFileDirectory; +import static net.consensys.linea.reporting.TestOutcomeWriterTool.getFileDirectory; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; @@ -31,7 +31,7 @@ public class BlockchainReferenceTestJson { @Synchronized public static CompletableFuture readBlockchainReferenceTestsOutput(String fileName) { - String fileDirectory = setFileDirectory(); + String fileDirectory = getFileDirectory(); return CompletableFuture.supplyAsync( () -> { Path directoryPath = Paths.get(fileDirectory); diff --git a/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java index 7c7fcfa770..7a7e6592f3 100644 --- a/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java +++ b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java @@ -14,24 +14,16 @@ */ package net.consensys.linea; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import static net.consensys.linea.reporting.TestOutcomeWriterTool.addFailure; + import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; -import net.consensys.linea.zktracer.json.JsonConverter; +import net.consensys.linea.reporting.TestOutcomeWriterTool; +import net.consensys.linea.reporting.TestState; @Slf4j public class ReferenceTestOutcomeRecorderTool { @@ -42,51 +34,23 @@ public class ReferenceTestOutcomeRecorderTool { public static final String JSON_OUTPUT_FILENAME = System.getenv() .getOrDefault("REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "failedReferenceTests.json"); - public static JsonConverter jsonConverter = JsonConverter.builder().build(); - private static volatile AtomicInteger failedCounter = new AtomicInteger(0); - private static volatile AtomicInteger successCounter = new AtomicInteger(0); - private static volatile AtomicInteger disabledCounter = new AtomicInteger(0); - private static volatile AtomicInteger abortedCounter = new AtomicInteger(0); - private static volatile ConcurrentMap< - String, ConcurrentMap>> - modulesToConstraintsToTests = new ConcurrentHashMap<>(); public static void mapAndStoreTestResult( String testName, TestState success, Map> failedConstraints) { switch (success) { case FAILED -> { - failedCounter.incrementAndGet(); + TestOutcomeWriterTool.addFailed(); for (Map.Entry> failedConstraint : failedConstraints.entrySet()) { String moduleName = failedConstraint.getKey(); for (String constraint : failedConstraint.getValue()) { - ConcurrentMap> constraintsToTests = - modulesToConstraintsToTests.computeIfAbsent( - moduleName, m -> new ConcurrentHashMap<>()); - ConcurrentSkipListSet failingTests = - constraintsToTests.computeIfAbsent(constraint, m -> new ConcurrentSkipListSet<>()); - int size = failingTests.size(); - failingTests.add(testName); - if (failingTests.size() == size) { - log.warn("Duplicate name found... {}", failedConstraint); - } + addFailure(moduleName, constraint, testName); } } } - case SUCCESS -> successCounter.incrementAndGet(); - case ABORTED -> abortedCounter.incrementAndGet(); - case DISABLED -> disabledCounter.incrementAndGet(); - } - } - - @Synchronized - public static BlockchainReferenceTestOutcome parseBlockchainReferenceTestOutcome( - String jsonString) { - if (!jsonString.isEmpty()) { - BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = - jsonConverter.fromJson(jsonString, BlockchainReferenceTestOutcome.class); - return blockchainReferenceTestOutcome; + case SUCCESS -> TestOutcomeWriterTool.addSuccess(); + case ABORTED -> TestOutcomeWriterTool.addAborted(); + case DISABLED -> TestOutcomeWriterTool.addSkipped(); } - throw new RuntimeException("invalid JSON"); } public static Map> extractConstraints(String message) { @@ -174,57 +138,5 @@ private static void getPairFromString(String constraint, Map pairs.computeIfAbsent(pair[0].trim(), p -> new HashSet<>()).add(pair[1].trim()); } - @Synchronized - public static void writeToJsonFile() { - try { - String directory = setFileDirectory(); - log.info("Reference test will be written to file {} \\ {}", directory, JSON_OUTPUT_FILENAME); - writeToJsonFileInternal(JSON_OUTPUT_FILENAME).get(); - log.info("Reference test results written to file {}", JSON_OUTPUT_FILENAME); - log.info( - "Path exists: {}, file exist: {}", - Paths.get(directory).toFile().exists(), - Paths.get(directory).resolve(JSON_OUTPUT_FILENAME).toFile().exists()); - } catch (Exception e) { - log.error("Error while writing results"); - throw new RuntimeException("Error while writing results", e); - } - } - static ObjectMapper objectMapper = new ObjectMapper(); - - @Synchronized - private static CompletableFuture writeToJsonFileInternal(String name) { - String fileDirectory = setFileDirectory(); - log.info("writing results summary to {}", fileDirectory + "/" + name); - try { - Files.createDirectories(Path.of(fileDirectory)); - } catch (IOException e) { - log.error("Error - Failed to create test directory output: %s".formatted(e.getMessage())); - throw new RuntimeException(e); - } - return CompletableFuture.runAsync( - () -> { - try (FileWriter file = new FileWriter(Path.of(fileDirectory, name).toString())) { - objectMapper.writeValue( - file, - new BlockchainReferenceTestOutcome( - failedCounter.get(), - successCounter.get(), - disabledCounter.get(), - abortedCounter.get(), - modulesToConstraintsToTests)); - } catch (Exception e) { - log.error("Error - Failed to write test output: %s".formatted(e.getMessage())); - } - }); - } - - static String setFileDirectory() { - String jsonDirectory = System.getenv("FAILED_TEST_JSON_DIRECTORY"); - if (jsonDirectory == null || jsonDirectory.isEmpty()) { - return "../tmp/local/"; - } - return jsonDirectory; - } } diff --git a/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeWriter.java b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeWriter.java index f745a6a827..84b4d9ca3c 100644 --- a/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeWriter.java +++ b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeWriter.java @@ -17,6 +17,7 @@ import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.*; import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.reporting.TestOutcomeWriterTool; import org.junit.platform.launcher.LauncherSession; import org.junit.platform.launcher.LauncherSessionListener; @@ -25,6 +26,6 @@ public class ReferenceTestOutcomeWriter implements LauncherSessionListener { @Override public void launcherSessionClosed(LauncherSession session) { - writeToJsonFile(); + TestOutcomeWriterTool.writeToJsonFile(JSON_OUTPUT_FILENAME); } } diff --git a/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java b/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java index 21929d4674..686d5cab02 100644 --- a/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java +++ b/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java @@ -36,6 +36,8 @@ import lombok.extern.slf4j.Slf4j; import net.consensys.linea.corset.CorsetValidator; +import net.consensys.linea.reporting.TestOutcome; +import net.consensys.linea.reporting.TestOutcomeWriterTool; import net.consensys.linea.testing.ExecutionEnvironment; import net.consensys.linea.zktracer.ZkTracer; import org.hyperledger.besu.ethereum.MainnetBlockValidator; @@ -94,6 +96,18 @@ public class BlockchainReferenceTestTools { PARAMS.ignore("RevertInCreateInInitCreate2_d0g0v0_London[London]"); PARAMS.ignore("RevertInCreateInInit_d0g0v0_London[London]"); + // Arithmetization restriction: recipient address is a precompile. + PARAMS.ignore("modexpRandomInput_d0g0v0_London[London]"); + PARAMS.ignore("modexpRandomInput_d0g1v0_London[London]"); + PARAMS.ignore("modexpRandomInput_d1g0v0_London[London]"); + PARAMS.ignore("modexpRandomInput_d1g1v0_London[London]"); + PARAMS.ignore("modexpRandomInput_d2g0v0_London[London]"); + PARAMS.ignore("modexpRandomInput_d2g1v0_London[London]"); + PARAMS.ignore("randomStatetest642_d0g0v0_London[London]"); + PARAMS.ignore("randomStatetest644_d0g0v0_London[London]"); + PARAMS.ignore("randomStatetest645_d0g0v0_London[London]"); + PARAMS.ignore("randomStatetest645_d0g0v1_London[London]"); + // Consumes a huge amount of memory. PARAMS.ignore("static_Call1MB1024Calldepth_d1g0v0_\\w+"); PARAMS.ignore("ShanghaiLove_.*"); @@ -105,6 +119,16 @@ public class BlockchainReferenceTestTools { // Absurd amount of gas, doesn't run in parallel. PARAMS.ignore("randomStatetest94_\\w+"); + // Balance is more than 128 bits + PARAMS.ignore("Call1024PreCalls_d0g0v0_London[London]"); + PARAMS.ignore("Call1024PreCalls_d0g1v0_London[London]"); + PARAMS.ignore("OverflowGasRequire_London[London]"); + PARAMS.ignore("StrangeContractCreation_London[London]"); + PARAMS.ignore("SuicideIssue_London[London]"); + PARAMS.ignore("DelegateCallSpam_London[London]"); + PARAMS.ignore("OverflowGasRequire2_d0g0v0_London[London]"); + PARAMS.ignore("HighGasLimit_d0g0v0_London[London]"); + // Don't do time-consuming tests. PARAMS.ignore("CALLBlake2f_MaxRounds.*"); PARAMS.ignore("loopMul_*"); @@ -129,9 +153,9 @@ public static CompletableFuture> getRecordedFailedTestsFromJson( return CompletableFuture.completedFuture(failedTests); } - CompletableFuture modulesToConstraintsFutures = + CompletableFuture modulesToConstraintsFutures = readBlockchainReferenceTestsOutput(JSON_INPUT_FILENAME) - .thenApply(ReferenceTestOutcomeRecorderTool::parseBlockchainReferenceTestOutcome); + .thenApply(TestOutcomeWriterTool::parseTestOutcome); return modulesToConstraintsFutures.thenApply( blockchainReferenceTestOutcome -> { diff --git a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java index 57a80c13e9..d22a7dbdef 100644 --- a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java +++ b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java @@ -25,6 +25,9 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListSet; +import net.consensys.linea.reporting.TestOutcome; +import net.consensys.linea.reporting.TestOutcomeWriterTool; +import net.consensys.linea.reporting.TestState; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -52,13 +55,13 @@ void multipleModulesAreStoredCorrectly() { "test1", TestState.FAILED, Map.of("Constraint", Set.of(module1, module2))); ReferenceTestOutcomeRecorderTool.mapAndStoreTestResult( "test2", TestState.FAILED, Map.of("Constraint", Set.of(module1))); - ReferenceTestOutcomeRecorderTool.writeToJsonFile(); + TestOutcomeWriterTool.writeToJsonFile(JSON_OUTPUT_FILENAME_TEST); readBlockchainReferenceTestsOutput(JSON_OUTPUT_FILENAME_TEST) .thenApply( jsonString -> { - BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = - ReferenceTestOutcomeRecorderTool.parseBlockchainReferenceTestOutcome(jsonString); + TestOutcome blockchainReferenceTestOutcome = + TestOutcomeWriterTool.parseTestOutcome(jsonString); ConcurrentMap>> modulesToConstraints = @@ -155,8 +158,8 @@ public void extractConstraints() { @Test void parseBlockchainReferenceTestOutcome() { - BlockchainReferenceTestOutcome outcome = - ReferenceTestOutcomeRecorderTool.parseBlockchainReferenceTestOutcome( + TestOutcome outcome = + TestOutcomeWriterTool.parseTestOutcome( """ { "abortedCounter": 20, diff --git a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java index 31bc358b39..2eb69c5722 100644 --- a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java +++ b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java @@ -14,7 +14,7 @@ */ package net.consensys.linea; -import static net.consensys.linea.TestState.*; +import static net.consensys.linea.reporting.TestState.*; import static net.consensys.linea.testing.ExecutionEnvironment.CORSET_VALIDATION_RESULT; import java.util.HashMap; diff --git a/testing/build.gradle b/testing/build.gradle index e1ef423795..aecee8c3b5 100644 --- a/testing/build.gradle +++ b/testing/build.gradle @@ -30,6 +30,7 @@ dependencies { implementation "${besuArtifactGroup}.internal:algorithms" implementation "${besuArtifactGroup}:plugin-api" + implementation 'org.junit.platform:junit-platform-launcher' implementation 'org.junit.jupiter:junit-jupiter-api' runtimeOnly 'org.junit.jupiter:junit-jupiter-engine' implementation 'org.junit.jupiter:junit-jupiter-params' diff --git a/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java b/testing/src/main/java/net/consensys/linea/reporting/TestOutcome.java similarity index 95% rename from reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java rename to testing/src/main/java/net/consensys/linea/reporting/TestOutcome.java index 9da7c9413f..e7e1067abb 100644 --- a/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java +++ b/testing/src/main/java/net/consensys/linea/reporting/TestOutcome.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package net.consensys.linea; +package net.consensys.linea.reporting; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListSet; @@ -35,7 +35,7 @@ "abortedCounter", "modulesToConstraintsToTests" }) -public class BlockchainReferenceTestOutcome { +public class TestOutcome { @JsonProperty private final int failedCounter; @JsonProperty private final int successCounter; @JsonProperty private final int disabledCounter; diff --git a/testing/src/main/java/net/consensys/linea/reporting/TestOutcomeWriterTool.java b/testing/src/main/java/net/consensys/linea/reporting/TestOutcomeWriterTool.java new file mode 100644 index 0000000000..1c38361bb0 --- /dev/null +++ b/testing/src/main/java/net/consensys/linea/reporting/TestOutcomeWriterTool.java @@ -0,0 +1,107 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea.reporting; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.atomic.AtomicInteger; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.zktracer.json.JsonConverter; + +@Slf4j +public class TestOutcomeWriterTool { + + public static void addFailed() { + failedCounter.incrementAndGet(); + } + + public static JsonConverter jsonConverter = JsonConverter.builder().build(); + private static ObjectMapper objectMapper = new ObjectMapper(); + + private static volatile AtomicInteger failedCounter = new AtomicInteger(0); + private static volatile AtomicInteger successCounter = new AtomicInteger(0); + private static volatile AtomicInteger disabledCounter = new AtomicInteger(0); + private static volatile AtomicInteger abortedCounter = new AtomicInteger(0); + private static volatile ConcurrentMap< + String, ConcurrentMap>> + modulesToConstraintsToTests = new ConcurrentHashMap<>(20); + + public static void addFailure(String type, String cause, String test) { + failedCounter.incrementAndGet(); + modulesToConstraintsToTests + .computeIfAbsent(type, t -> new ConcurrentHashMap<>()) + .computeIfAbsent(cause, t -> new ConcurrentSkipListSet<>()) + .add(test); + } + + public static void addSuccess() { + successCounter.incrementAndGet(); + } + + public static void addAborted() { + abortedCounter.incrementAndGet(); + } + + public static void addSkipped() { + disabledCounter.incrementAndGet(); + } + + public static void writeToJsonFile(String name) { + String fileDirectory = getFileDirectory(); + log.info("writing results summary to {}", fileDirectory + "/" + name); + try { + Files.createDirectories(Path.of(fileDirectory)); + } catch (IOException e) { + log.error("Error - Failed to create test directory output: %s".formatted(e.getMessage())); + throw new RuntimeException(e); + } + try (FileWriter file = new FileWriter(Path.of(fileDirectory, name).toString())) { + objectMapper.writeValue( + file, + new TestOutcome( + failedCounter.get(), + successCounter.get(), + disabledCounter.get(), + abortedCounter.get(), + modulesToConstraintsToTests)); + } catch (Exception e) { + log.error("Error - Failed to write test output: %s".formatted(e.getMessage())); + } + } + + public static String getFileDirectory() { + String jsonDirectory = System.getenv("FAILED_TEST_JSON_DIRECTORY"); + if (jsonDirectory == null || jsonDirectory.isEmpty()) { + return "../tmp/local/"; + } + return jsonDirectory; + } + + public static TestOutcome parseTestOutcome(String jsonString) { + if (!jsonString.isEmpty()) { + TestOutcome blockchainReferenceTestOutcome = + jsonConverter.fromJson(jsonString, TestOutcome.class); + return blockchainReferenceTestOutcome; + } + throw new RuntimeException("invalid JSON"); + } +} diff --git a/reference-tests/src/main/java/net/consensys/linea/TestState.java b/testing/src/main/java/net/consensys/linea/reporting/TestState.java similarity index 94% rename from reference-tests/src/main/java/net/consensys/linea/TestState.java rename to testing/src/main/java/net/consensys/linea/reporting/TestState.java index af39139938..ca01b606d9 100644 --- a/reference-tests/src/main/java/net/consensys/linea/TestState.java +++ b/testing/src/main/java/net/consensys/linea/reporting/TestState.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package net.consensys.linea; +package net.consensys.linea.reporting; public enum TestState { DISABLED,