Export EOF validation unit tests to json EOFTests (#818)

lib/evmone/eof.cpp

@@ -667,12 +667,8 @@ std::string_view get_error_message(EOFValidationError err) noexcept
     {
     case EOFValidationError::success:
         return "success";
-    case EOFValidationError::starts_with_format:
-        return "starts_with_format";
     case EOFValidationError::invalid_prefix:
         return "invalid_prefix";
-    case EOFValidationError::eof_version_mismatch:
-        return "eof_version_mismatch";
     case EOFValidationError::eof_version_unknown:
         return "eof_version_unknown";
     case EOFValidationError::incomplete_section_size:

lib/evmone/eof.hpp

@@ -68,9 +68,7 @@ struct EOF1Header
 enum class EOFValidationError
 {
     success,
-    starts_with_format,
     invalid_prefix,
-    eof_version_mismatch,
     eof_version_unknown,
 
     incomplete_section_size,

test/integration/statetest/CMakeLists.txt

@@ -143,9 +143,28 @@ set_tests_properties(
 add_test(
     NAME ${PREFIX}/execute_exported_tests
     # TODO: Broken exported tests are filtered out.
-    COMMAND evmone-statetest ${EXPORT_DIR} --gtest_filter=-*block.*:*tx.tx_non_existing_sender
+    COMMAND evmone-statetest ${EXPORT_DIR}/state_transition --gtest_filter=-*block.*:*tx.tx_non_existing_sender
 )
 set_tests_properties(
     ${PREFIX}/execute_exported_tests PROPERTIES
     FIXTURES_REQUIRED EXPORT_JSON_TESTS
 )
+
+add_test(
+    NAME ${PREFIX}/export_validation_tests_to_json
+    COMMAND evmone-unittests --gtest_filter=eof_validation.*
+)
+set_tests_properties(
+    ${PREFIX}/export_validation_tests_to_json PROPERTIES
+    ENVIRONMENT EVMONE_EXPORT_TESTS=${EXPORT_DIR}
+    FIXTURES_SETUP EXPORT_JSON_VALIDATION_TESTS
+)
+
+add_test(
+    NAME ${PREFIX}/execute_exported_validation_tests
+    COMMAND evmone-eoftest ${EXPORT_DIR}/eof_validation
+)
+set_tests_properties(
+    ${PREFIX}/execute_exported_validation_tests PROPERTIES
+    FIXTURES_REQUIRED EXPORT_JSON_VALIDATION_TESTS
+)

test/unittests/eof_validation.cpp

@@ -3,9 +3,90 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "eof_validation.hpp"
+#include <test/statetest/statetest.hpp>
+#include <fstream>
 
 namespace evmone::test
 {
+namespace
+{
+std::string_view get_tests_error_message(EOFValidationError err) noexcept
+{
+    switch (err)
+    {
+    case EOFValidationError::success:
+        return "success";
+    case EOFValidationError::invalid_prefix:
+        return "EOF_InvalidPrefix";
+    case EOFValidationError::eof_version_unknown:
+        return "EOF_UnknownVersion";
+    case EOFValidationError::incomplete_section_size:
+        return "EOF_IncompleteSectionSize";
+    case EOFValidationError::incomplete_section_number:
+        return "EOF_IncompleteSectionNumber";
+    case EOFValidationError::header_terminator_missing:
+        return "EOF_HeaderTerminatorMissing";
+    case EOFValidationError::type_section_missing:
+        return "EOF_TypeSectionMissing";
+    case EOFValidationError::code_section_missing:
+        return "EOF_CodeSectionMissing";
+    case EOFValidationError::data_section_missing:
+        return "EOF_DataSectionMissing";
+    case EOFValidationError::zero_section_size:
+        return "EOF_InvalidSectionSize";
+    case EOFValidationError::section_headers_not_terminated:
+        return "EOF_SectionHeadersNotTerminated";
+    case EOFValidationError::invalid_section_bodies_size:
+        return "EOF_InvalidSectionBodiesSize";
+    case EOFValidationError::unreachable_code_sections:
+        return "EOF_UnreachableCodeSections";
+    case EOFValidationError::undefined_instruction:
+        return "EOF_UndefinedInstruction";
+    case EOFValidationError::truncated_instruction:
+        return "EOF_TruncatedInstruction";
+    case EOFValidationError::invalid_rjump_destination:
+        return "EOF_InvalidJumpDestination";
+    case EOFValidationError::too_many_code_sections:
+        return "EOF_TooManyCodeSections";
+    case EOFValidationError::invalid_type_section_size:
+        return "EOF_InvalidTypeSectionSize";
+    case EOFValidationError::invalid_first_section_type:
+        return "EOF_InvalidFirstSectionType";
+    case EOFValidationError::invalid_max_stack_height:
+        return "EOF_InvalidMaxStackHeight";
+    case EOFValidationError::max_stack_height_above_limit:
+        return "EOF_MaxStackHeightExceeded";
+    case EOFValidationError::inputs_outputs_num_above_limit:
+        return "EOF_InputsOutputsAboveLimit";
+    case EOFValidationError::no_terminating_instruction:
+        return "EOF_InvalidCodeTermination";
+    case EOFValidationError::stack_height_mismatch:
+        return "EOF_ConflictingStackHeight";
+    case EOFValidationError::stack_higher_than_outputs_required:
+        return "EOF_InvalidNumberOfOutputs";
+    case EOFValidationError::unreachable_instructions:
+        return "EOF_UnreachableCode";
+    case EOFValidationError::stack_underflow:
+        return "EOF_StackUnderflow";
+    case EOFValidationError::stack_overflow:
+        return "EOF_StackOverflow";
+    case EOFValidationError::invalid_code_section_index:
+        return "EOF_InvalidCodeSectionIndex";
+    case EOFValidationError::invalid_dataloadn_index:
+        return "EOF_InvalidDataloadnIndex";
+    case EOFValidationError::jumpf_destination_incompatible_outputs:
+        return "EOF_JumpfDestinationIncompatibleOutputs";
+    case EOFValidationError::invalid_non_returning_flag:
+        return "EOF_InvalidNonReturningFlag";
+    case EOFValidationError::callf_to_non_returning_function:
+        return "EOF_CallfToNonReturning";
+    case EOFValidationError::impossible:
+        return "impossible";
+    }
+    return "<unknown>";
+}
+}  // namespace
+
 void eof_validation::TearDown()
 {
     for (size_t i = 0; i < test_cases.size(); ++i)
@@ -15,5 +96,37 @@ void eof_validation::TearDown()
             << "test case " << i << " " << test_case.name << "\n"
             << hex(test_case.container);
     }
+
+    if (!export_file_path.empty())
+        export_eof_validation_test();
+}
+
+void eof_validation::export_eof_validation_test()
+{
+    json::json j;
+    auto& jt = j[export_test_name];
+
+    auto& jvectors = jt["vectors"];
+    for (size_t i = 0; i < test_cases.size(); ++i)
+    {
+        const auto& test_case = test_cases[i];
+        const auto case_name = test_case.name.empty() ?
+                                   (std::string{export_test_name} + "_" + std::to_string(i)) :
+                                   test_case.name;
+
+        auto& jcase = jvectors[case_name];
+        jcase["code"] = hex0x(test_case.container);
+
+        auto& jresults = jcase["results"][evmc::to_string(rev)];
+        if (test_case.error == EOFValidationError::success)
+            jresults["result"] = true;
+        else
+        {
+            jresults["result"] = false;
+            jresults["exception"] = get_tests_error_message(test_case.error);
+        }
+    }
+
+    std::ofstream{export_file_path} << std::setw(2) << j;
 }
 }  // namespace evmone::test

test/unittests/eof_validation.hpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 #pragma once
 
+#include "exportable_fixture.hpp"
 #include <evmc/evmc.hpp>
 #include <evmone/eof.hpp>
 #include <evmone/evmone.h>
@@ -16,7 +17,7 @@ using evmc::bytes;
 /// Fixture for defining test cases for EOF validation.
 ///
 /// Each test contains multiple cases, which are validated during test teardown.
-class eof_validation : public testing::Test
+class eof_validation : public ExportableFixture
 {
 protected:
     /// EOF validation test case.
@@ -28,6 +29,7 @@ class eof_validation : public testing::Test
         /// or EOFValidationError::success if it is expected to be valid.
         EOFValidationError error = EOFValidationError::success;
         /// (Optional) Test case description.
+        /// In non-empty, exported test file will use it for test case name.
         std::string name;
     };
 
@@ -45,6 +47,9 @@ class eof_validation : public testing::Test
 
     /// The test runner.
     void TearDown() override;
+
+    /// Exports the test in the JSON EOF Test format in the given directory.
+    void export_eof_validation_test();
 };
 
 }  // namespace evmone::test