EOFTests

circle.yml

@@ -486,13 +486,18 @@ jobs:
             ~/tests/EIPTests/StateTests/stEIP1153-transientStorage
       - download_execution_tests:
           repo: ipsilon/tests
-          rev: eof-rjumpv-20230803
+          rev: eof-update-20230828
           legacy: false
       - run:
           name: "State tests (EOF)"
           working_directory: ~/build
           command: |
             bin/evmone-statetest ~/tests/EIPTests/StateTests/stEOF
+      - run:
+          name: "EOF validation tests"
+          working_directory: ~/build
+          command: |
+            bin/evmone-eoftest ~/tests/EOFTests
       - collect_coverage_gcc
       - upload_coverage:
           flags: statetests

test/CMakeLists.txt

@@ -20,10 +20,11 @@ add_subdirectory(integration)
 add_subdirectory(internal_benchmarks)
 add_subdirectory(state)
 add_subdirectory(statetest)
+add_subdirectory(eoftest)
 add_subdirectory(t8n)
 add_subdirectory(unittests)
 
-set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-t8n evmone-unittests)
+set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-eoftest evmone-t8n evmone-unittests)
 
 if(EVMONE_FUZZING)
     add_subdirectory(eofparsefuzz)

test/eoftest/.clang-tidy

@@ -0,0 +1,3 @@
+InheritParentConfig: true
+Checks: >
+  -clang-analyzer-cplusplus.NewDeleteLeaks

test/eoftest/CMakeLists.txt

@@ -0,0 +1,15 @@
+# evmone: Fast Ethereum Virtual Machine implementation
+# Copyright 2023 The evmone Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+hunter_add_package(nlohmann_json)
+find_package(nlohmann_json CONFIG REQUIRED)
+
+add_executable(evmone-eoftest)
+target_link_libraries(evmone-eoftest PRIVATE evmone evmone::testutils nlohmann_json::nlohmann_json GTest::gtest)
+target_include_directories(evmone-eoftest PRIVATE ${evmone_private_include_dir})
+target_sources(
+    evmone-eoftest PRIVATE
+    eoftest.cpp
+    eoftest_runner.cpp
+)

test/eoftest/eoftest.cpp

@@ -0,0 +1,87 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+
+#include "eoftest.hpp"
+#include <CLI/CLI.hpp>
+#include <evmone/evmone.h>
+#include <gtest/gtest.h>
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace
+{
+
+class EOFTest : public testing::Test
+{
+    fs::path m_json_test_file;
+
+public:
+    explicit EOFTest(fs::path json_test_file) noexcept : m_json_test_file{std::move(json_test_file)}
+    {}
+
+    void TestBody() final
+    {
+        std::ifstream f{m_json_test_file};
+        evmone::test::run_eof_test(f);
+    }
+};
+
+void register_test(const std::string& suite_name, const fs::path& file)
+{
+    testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
+        file.string().c_str(), 0, [file]() -> testing::Test* { return new EOFTest(file); });
+}
+
+void register_test_files(const fs::path& root)
+{
+    if (is_directory(root))
+    {
+        std::vector<fs::path> test_files;
+        std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{},
+            std::back_inserter(test_files), [](const fs::directory_entry& entry) {
+                return entry.is_regular_file() && entry.path().extension() == ".json";
+            });
+        std::sort(test_files.begin(), test_files.end());
+
+        for (const auto& p : test_files)
+            register_test(fs::relative(p, root).parent_path().string(), p);
+    }
+    else  // Treat as a file.
+    {
+        register_test(root.parent_path().string(), root);
+    }
+}
+
+}  // namespace
+
+
+int main(int argc, char* argv[])
+{
+    try
+    {
+        testing::InitGoogleTest(&argc, argv);
+        CLI::App app{"evmone eof test runner"};
+
+        std::vector<std::string> paths;
+        app.add_option("path", paths, "Path to test file or directory")
+            ->required()
+            ->check(CLI::ExistingPath);
+
+        CLI11_PARSE(app, argc, argv);
+
+        for (const auto& p : paths)
+        {
+            register_test_files(p);
+        }
+
+        return RUN_ALL_TESTS();
+    }
+    catch (const std::exception& ex)
+    {
+        std::cerr << ex.what() << "\n";
+        return -1;
+    }
+}

test/eoftest/eoftest.hpp

@@ -0,0 +1,13 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include <iosfwd>
+
+namespace evmone::test
+{
+
+void run_eof_test(std::istream& input);
+
+}  // namespace evmone::test

test/eoftest/eoftest_runner.cpp

@@ -0,0 +1,78 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "../utils/utils.hpp"
+#include "eoftest.hpp"
+#include <evmc/evmc.hpp>
+#include <evmone/eof.hpp>
+#include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
+
+namespace json = nlohmann;
+
+namespace evmone::test
+{
+
+namespace
+{
+struct EOFValidationTest
+{
+    struct Case
+    {
+        struct Expectation
+        {
+            evmc_revision rev;
+            bool result;
+        };
+        std::string name;
+        evmc::bytes code;
+        std::vector<Expectation> expectations;
+    };
+    std::unordered_map<std::string, Case> cases;
+};
+
+void from_json(const json::json& j, EOFValidationTest::Case& o)
+{
+    const auto op_code = evmc::from_hex(j.at("code").get<std::string>());
+    if (!op_code)
+        throw std::invalid_argument{"code is invalid hex string"};
+    o.code = *op_code;
+
+    for (const auto& [rev, result] : j.at("results").items())
+    {
+        o.expectations.push_back({to_rev(rev), result.at("result").get<bool>()});
+    }
+}
+
+void from_json(const json::json& j, EOFValidationTest& o)
+{
+    if (!j.is_object() || j.empty())
+        throw std::invalid_argument{"JSON test must be an object with single key of the test name"};
+
+    const auto& j_t = *j.begin();  // Content is in a dict with the test name.
+
+    for (const auto& [name, test] : j_t.at("vectors").items())
+    {
+        o.cases.emplace(name, test.get<EOFValidationTest::Case>());
+    }
+}
+
+}  // namespace
+
+void run_eof_test(std::istream& input)
+{
+    const auto test = json::json::parse(input).get<EOFValidationTest>();
+    for (const auto& [name, cases] : test.cases)
+    {
+        for (const auto& expectation : cases.expectations)
+        {
+            const auto result = evmone::validate_eof(expectation.rev, cases.code);
+            const bool b_result = (result == EOFValidationError::success);
+            EXPECT_EQ(b_result, expectation.result)
+                << name << " " << expectation.rev << " " << hex(cases.code);
+        }
+    }
+}
+
+}  // namespace evmone::test

test/statetest/CMakeLists.txt

@@ -8,7 +8,7 @@ find_package(nlohmann_json CONFIG REQUIRED)
 add_library(evmone-statetestutils STATIC)
 add_library(evmone::statetestutils ALIAS evmone-statetestutils)
 target_compile_features(evmone-statetestutils PUBLIC cxx_std_20)
-target_link_libraries(evmone-statetestutils PRIVATE evmone::state nlohmann_json::nlohmann_json)
+target_link_libraries(evmone-statetestutils PRIVATE evmone::state evmone::testutils nlohmann_json::nlohmann_json)
 target_include_directories(evmone-statetestutils PRIVATE ${evmone_private_include_dir})
 target_sources(
     evmone-statetestutils PRIVATE

test/statetest/statetest.hpp

@@ -13,9 +13,6 @@ namespace json = nlohmann;
 namespace evmone::test
 {
 
-/// Translates tests fork name to EVM revision
-evmc_revision to_rev(std::string_view s);
-
 struct TestMultiTransaction : state::Transaction
 {
     struct Indexes

test/statetest/statetest_loader.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "../utils/stdx/utility.hpp"
+#include "../utils/utils.hpp"
 #include "statetest.hpp"
 #include <evmone/eof.hpp>
 #include <nlohmann/json.hpp>
@@ -252,41 +253,6 @@ state::State from_json<state::State>(const json::json& j)
     return o;
 }
 
-evmc_revision to_rev(std::string_view s)
-{
-    if (s == "Frontier")
-        return EVMC_FRONTIER;
-    if (s == "Homestead")
-        return EVMC_HOMESTEAD;
-    if (s == "EIP150")
-        return EVMC_TANGERINE_WHISTLE;
-    if (s == "EIP158")
-        return EVMC_SPURIOUS_DRAGON;
-    if (s == "Byzantium")
-        return EVMC_BYZANTIUM;
-    if (s == "Constantinople")
-        return EVMC_CONSTANTINOPLE;
-    if (s == "ConstantinopleFix")
-        return EVMC_PETERSBURG;
-    if (s == "Istanbul")
-        return EVMC_ISTANBUL;
-    if (s == "Berlin")
-        return EVMC_BERLIN;
-    if (s == "London" || s == "ArrowGlacier")
-        return EVMC_LONDON;
-    if (s == "Merge")
-        return EVMC_PARIS;
-    if (s == "Merge+3855")  // PUSH0
-        return EVMC_SHANGHAI;
-    if (s == "Shanghai")
-        return EVMC_SHANGHAI;
-    if (s == "Cancun")
-        return EVMC_CANCUN;
-    if (s == "Prague")
-        return EVMC_PRAGUE;
-    throw std::invalid_argument{"unknown revision: " + std::string{s}};
-}
-
 /// Load common parts of Transaction or TestMultiTransaction.
 static void from_json_tx_common(const json::json& j, state::Transaction& o)
 {

test/t8n/t8n.cpp

@@ -7,6 +7,7 @@
 #include "../state/mpt_hash.hpp"
 #include "../state/rlp.hpp"
 #include "../statetest/statetest.hpp"
+#include "../utils/utils.hpp"
 #include <evmone/evmone.h>
 #include <evmone/version.h>
 #include <nlohmann/json.hpp>
@@ -47,7 +48,7 @@ int main(int argc, const char* argv[])
                 return 0;
             }
             if (arg == "--state.fork" && ++i < argc)
-                rev = evmone::test::to_rev(argv[i]);
+                rev = to_rev(argv[i]);
             else if (arg == "--input.alloc" && ++i < argc)
                 alloc_file = argv[i];
             else if (arg == "--input.env" && ++i < argc)

test/utils/CMakeLists.txt

@@ -4,9 +4,9 @@
 
 find_package(intx CONFIG REQUIRED)
 
-add_library(testutils INTERFACE)
+add_library(testutils STATIC)
 add_library(evmone::testutils ALIAS testutils)
-target_link_libraries(testutils INTERFACE evmc::evmc_cpp)
+target_link_libraries(testutils PUBLIC evmc::evmc_cpp)
 target_include_directories(testutils INTERFACE ${PROJECT_SOURCE_DIR} ${evmone_private_include_dir})
 
 if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19)
@@ -17,3 +17,4 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19)
         utils.hpp
     )
 endif()
+target_sources(testutils PRIVATE utils.cpp)

test/utils/utils.cpp

@@ -0,0 +1,45 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "utils.hpp"
+
+namespace evmone::test
+{
+
+evmc_revision to_rev(std::string_view s)
+{
+    if (s == "Frontier")
+        return EVMC_FRONTIER;
+    if (s == "Homestead")
+        return EVMC_HOMESTEAD;
+    if (s == "EIP150")
+        return EVMC_TANGERINE_WHISTLE;
+    if (s == "EIP158")
+        return EVMC_SPURIOUS_DRAGON;
+    if (s == "Byzantium")
+        return EVMC_BYZANTIUM;
+    if (s == "Constantinople")
+        return EVMC_CONSTANTINOPLE;
+    if (s == "ConstantinopleFix")
+        return EVMC_PETERSBURG;
+    if (s == "Istanbul")
+        return EVMC_ISTANBUL;
+    if (s == "Berlin")
+        return EVMC_BERLIN;
+    if (s == "London" || s == "ArrowGlacier")
+        return EVMC_LONDON;
+    if (s == "Merge")
+        return EVMC_PARIS;
+    if (s == "Merge+3855")  // PUSH0
+        return EVMC_SHANGHAI;
+    if (s == "Shanghai")
+        return EVMC_SHANGHAI;
+    if (s == "Cancun")
+        return EVMC_CANCUN;
+    if (s == "Prague")
+        return EVMC_PRAGUE;
+    throw std::invalid_argument{"unknown revision: " + std::string{s}};
+}
+
+}  // namespace evmone::test