Add eoftest tool to run EOF validation tests of Ethereum Test Suite

Co-authored-by: Andrei Maiboroda <andrei@ethereum.org>

circle.yml

@@ -481,6 +481,11 @@ jobs:
           working_directory: ~/build
           command: |
             bin/evmone-statetest ~/tests/EIPTests/StateTests/stEOF
+      - run:
+          name: "EOF validation tests"
+          working_directory: ~/build
+          command: |
+            bin/evmone-eoftest ~/tests/EOFTests
       - collect_coverage_gcc
       - upload_coverage:
           flags: statetests

test/CMakeLists.txt

@@ -20,10 +20,11 @@ add_subdirectory(integration)
 add_subdirectory(internal_benchmarks)
 add_subdirectory(state)
 add_subdirectory(statetest)
+add_subdirectory(eoftest)
 add_subdirectory(t8n)
 add_subdirectory(unittests)
 
-set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-t8n evmone-unittests)
+set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-eoftest evmone-t8n evmone-unittests)
 
 if(EVMONE_FUZZING)
     add_subdirectory(eofparsefuzz)

test/eoftest/.clang-tidy

@@ -0,0 +1,3 @@
+InheritParentConfig: true
+Checks: >
+  -clang-analyzer-cplusplus.NewDeleteLeaks

test/eoftest/CMakeLists.txt

@@ -0,0 +1,15 @@
+# evmone: Fast Ethereum Virtual Machine implementation
+# Copyright 2023 The evmone Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+hunter_add_package(nlohmann_json)
+find_package(nlohmann_json CONFIG REQUIRED)
+
+add_executable(evmone-eoftest)
+target_link_libraries(evmone-eoftest PRIVATE evmone evmone::testutils nlohmann_json::nlohmann_json GTest::gtest)
+target_include_directories(evmone-eoftest PRIVATE ${evmone_private_include_dir})
+target_sources(
+    evmone-eoftest PRIVATE
+    eoftest.cpp
+    eoftest_runner.cpp
+)

test/eoftest/eoftest.cpp

@@ -0,0 +1,87 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+
+#include "eoftest.hpp"
+#include <CLI/CLI.hpp>
+#include <evmone/evmone.h>
+#include <gtest/gtest.h>
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace
+{
+
+class EOFTest : public testing::Test
+{
+    fs::path m_json_test_file;
+
+public:
+    explicit EOFTest(fs::path json_test_file) noexcept : m_json_test_file{std::move(json_test_file)}
+    {}
+
+    void TestBody() final
+    {
+        std::ifstream f{m_json_test_file};
+        evmone::test::run_eof_test(f);
+    }
+};
+
+void register_test(const std::string& suite_name, const fs::path& file)
+{
+    testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
+        file.string().c_str(), 0, [file]() -> testing::Test* { return new EOFTest(file); });
+}
+
+void register_test_files(const fs::path& root)
+{
+    if (is_directory(root))
+    {
+        std::vector<fs::path> test_files;
+        std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{},
+            std::back_inserter(test_files), [](const fs::directory_entry& entry) {
+                return entry.is_regular_file() && entry.path().extension() == ".json";
+            });
+        std::sort(test_files.begin(), test_files.end());
+
+        for (const auto& p : test_files)
+            register_test(fs::relative(p, root).parent_path().string(), p);
+    }
+    else  // Treat as a file.
+    {
+        register_test(root.parent_path().string(), root);
+    }
+}
+
+}  // namespace
+
+
+int main(int argc, char* argv[])
+{
+    try
+    {
+        testing::InitGoogleTest(&argc, argv);
+        CLI::App app{"evmone eof test runner"};
+
+        std::vector<std::string> paths;
+        app.add_option("path", paths, "Path to test file or directory")
+            ->required()
+            ->check(CLI::ExistingPath);
+
+        CLI11_PARSE(app, argc, argv);
+
+        for (const auto& p : paths)
+        {
+            register_test_files(p);
+        }
+
+        return RUN_ALL_TESTS();
+    }
+    catch (const std::exception& ex)
+    {
+        std::cerr << ex.what() << "\n";
+        return -1;
+    }
+}

test/eoftest/eoftest.hpp

@@ -0,0 +1,13 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include <iosfwd>
+
+namespace evmone::test
+{
+
+void run_eof_test(std::istream& input);
+
+}  // namespace evmone::test

test/eoftest/eoftest_runner.cpp

@@ -0,0 +1,78 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "../utils/utils.hpp"
+#include "eoftest.hpp"
+#include <evmc/evmc.hpp>
+#include <evmone/eof.hpp>
+#include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
+
+namespace json = nlohmann;
+
+namespace evmone::test
+{
+
+namespace
+{
+struct EOFValidationTest
+{
+    struct Case
+    {
+        struct Expectation
+        {
+            evmc_revision rev;
+            bool result;
+        };
+        std::string name;
+        evmc::bytes code;
+        std::vector<Expectation> expectations;
+    };
+    std::unordered_map<std::string, Case> cases;
+};
+
+void from_json(const json::json& j, EOFValidationTest::Case& o)
+{
+    const auto op_code = evmc::from_hex(j.at("code").get<std::string>());
+    if (!op_code)
+        throw std::invalid_argument{"code is invalid hex string"};
+    o.code = *op_code;
+
+    for (const auto& [rev, result] : j.at("results").items())
+    {
+        o.expectations.push_back({to_rev(rev), result.at("result").get<bool>()});
+    }
+}
+
+void from_json(const json::json& j, EOFValidationTest& o)
+{
+    if (!j.is_object() || j.empty())
+        throw std::invalid_argument{"JSON test must be an object with single key of the test name"};
+
+    const auto& j_t = *j.begin();  // Content is in a dict with the test name.
+
+    for (const auto& [name, test] : j_t.at("vectors").items())
+    {
+        o.cases.emplace(name, test.get<EOFValidationTest::Case>());
+    }
+}
+
+}  // namespace
+
+void run_eof_test(std::istream& input)
+{
+    const auto test = json::json::parse(input).get<EOFValidationTest>();
+    for (const auto& [name, cases] : test.cases)
+    {
+        for (const auto& expectation : cases.expectations)
+        {
+            const auto result = evmone::validate_eof(expectation.rev, cases.code);
+            const bool b_result = (result == EOFValidationError::success);
+            EXPECT_EQ(b_result, expectation.result)
+                << name << " " << expectation.rev << " " << hex(cases.code);
+        }
+    }
+}
+
+}  // namespace evmone::test