Tests for EOF implementation. Note: The exception field of the json i…

…s not being compared.

circle.yml

@@ -474,13 +474,18 @@ jobs:
             bin/evmone-statetest ~/tests/GeneralStateTests ~/tests/LegacyTests/Constantinople/GeneralStateTests
       - download_execution_tests:
           repo: ipsilon/tests
-          rev: eof-rjumpv-20230803
+          rev: update-tests
           legacy: false
       - run:
           name: "State tests (EOF)"
           working_directory: ~/build
           command: |
             bin/evmone-statetest ~/tests/EIPTests/StateTests/stEOF
+      - run:
+          name: "EOF validation tests"
+          working_directory: ~/build
+          command: |
+            bin/evmone-eoftests ~/tests/EOFTests
       - collect_coverage_gcc
       - upload_coverage:
           flags: statetests

test/CMakeLists.txt

@@ -20,10 +20,11 @@ add_subdirectory(integration)
 add_subdirectory(internal_benchmarks)
 add_subdirectory(state)
 add_subdirectory(statetest)
+add_subdirectory(eoftests)
 add_subdirectory(t8n)
 add_subdirectory(unittests)
 
-set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-t8n evmone-unittests)
+set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-eoftests evmone-t8n evmone-unittests)
 
 if(EVMONE_FUZZING)
     add_subdirectory(eofparsefuzz)

test/eoftests/.clang-tidy

@@ -0,0 +1,3 @@
+InheritParentConfig: true
+Checks: >
+  -clang-analyzer-cplusplus.NewDeleteLeaks

test/eoftests/CMakeLists.txt

@@ -0,0 +1,15 @@
+# evmone: Fast Ethereum Virtual Machine implementation
+# Copyright 2023 The evmone Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+hunter_add_package(nlohmann_json)
+find_package(nlohmann_json CONFIG REQUIRED)
+
+add_executable(evmone-eoftests)
+target_link_libraries(evmone-eoftests PRIVATE evmone nlohmann_json::nlohmann_json GTest::gtest)
+target_include_directories(evmone-eoftests PRIVATE ${evmone_private_include_dir})
+target_sources(
+    evmone-eoftests PRIVATE
+    eoftests.cpp
+    eoftests_runner.cpp
+)

test/eoftests/eoftests.cpp

@@ -0,0 +1,82 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2022 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "eoftests.hpp"
+#include <CLI/CLI.hpp>
+#include <evmone/evmone.h>
+#include <gtest/gtest.h>
+#include <iostream>
+
+namespace
+{
+class EOFTest : public testing::Test
+{
+    fs::path m_json_test_file;
+
+public:
+    explicit EOFTest(fs::path json_test_file) noexcept : m_json_test_file{std::move(json_test_file)}
+    {}
+
+    void TestBody() final
+    {
+        std::ifstream f{m_json_test_file};
+        evmone::test::run_eof_test(f);
+    }
+};
+
+void register_test(const std::string& suite_name, const fs::path& file)
+{
+    testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
+        file.string().c_str(), 0, [file]() -> testing::Test* { return new EOFTest(file); });
+}
+
+void register_test_files(const fs::path& root)
+{
+    if (is_directory(root))
+    {
+        std::vector<fs::path> test_files;
+        std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{},
+            std::back_inserter(test_files), [](const fs::directory_entry& entry) {
+                return entry.is_regular_file() && entry.path().extension() == ".json";
+            });
+        std::sort(test_files.begin(), test_files.end());
+
+        for (const auto& p : test_files)
+            register_test(fs::relative(p, root).parent_path().string(), p);
+    }
+    else  // Treat as a file.
+    {
+        register_test(root.parent_path().string(), root);
+    }
+}
+}  // namespace
+
+
+int main(int argc, char* argv[])
+{
+    try
+    {
+        testing::InitGoogleTest(&argc, argv);
+        CLI::App app{"evmone eof test runner"};
+
+        std::vector<std::string> paths;
+        app.add_option("path", paths, "Path to test file or directory")
+            ->required()
+            ->check(CLI::ExistingPath);
+
+        CLI11_PARSE(app, argc, argv);
+
+        for (const auto& p : paths)
+        {
+            register_test_files(p);
+        }
+
+        return RUN_ALL_TESTS();
+    }
+    catch (const std::exception& ex)
+    {
+        std::cerr << ex.what() << "\n";
+        return -1;
+    }
+}

test/eoftests/eoftests.hpp

@@ -0,0 +1,40 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2022 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+
+#include <evmc/evmc.hpp>
+#include <evmone/eof.hpp>
+#include <nlohmann/json.hpp>
+#include <filesystem>
+
+namespace fs = std::filesystem;
+namespace json = nlohmann;
+
+namespace evmone::test
+{
+struct EOFTests
+{
+    struct Case
+    {
+        struct Expectation
+        {
+            evmc_revision rev;
+            bool result;
+            std::string exception;
+        };
+        std::string name;
+        evmc::bytes code;
+        std::vector<Expectation> expectations;
+    };
+    std::unordered_map<std::string, Case> cases;
+};
+
+void run_eof_test(std::istream& input);
+
+void from_json(const json::json& j, EOFTests& o);
+
+void from_json(const json::json& j, EOFTests::Case& o);
+
+}  // namespace evmone::test

test/eoftests/eoftests_runner.cpp

@@ -0,0 +1,61 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2022 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "../utils/utils.hpp"
+#include "eoftests.hpp"
+#include <evmc/evmc.hpp>
+#include <evmone/eof.hpp>
+#include <gtest/gtest.h>
+
+
+namespace evmone::test
+{
+namespace json = nlohmann;
+void run_eof_test(std::istream& input)
+{
+    const EOFTests test = json::json::parse(input).get<EOFTests>();
+    for (const auto& [name, cases] : test.cases)
+    {
+        for (const auto& expectation : cases.expectations)
+        {
+            const EOFValidationError result = evmone::validate_eof(expectation.rev, cases.code);
+            const bool b_result = (result == EOFValidationError::success);
+            EXPECT_EQ(b_result, expectation.result) << expectation.rev << " " << hex(cases.code);
+        }
+    }
+}
+
+
+void from_json(const json::json& j, EOFTests& o)
+{
+    if (!j.is_object() || j.empty())
+        throw std::invalid_argument{"JSON test must be an object with single key of the test name"};
+
+    const auto& j_t = *j.begin();  // Content is in a dict with the test name.
+
+    for (const auto& [name, test] : j_t.at("vectors").items())
+    {
+        o.cases.insert({name, test.get<EOFTests::Case>()});
+    }
+}
+
+
+void from_json(const json::json& j, EOFTests::Case& o)
+{
+    std::optional<evmc::bytes> hex_str{evmc::from_hex(j.at("code").get<std::string>())};
+    if (!hex_str)
+        throw std::invalid_argument{"invalid hex string"};
+    const evmc::bytes code{hex_str.value()};
+    o.code = code;
+
+    for (const auto& [rev, result] : j.at("results").items())
+    {
+        EOFTests::Case::Expectation expectation{};
+        expectation.rev = to_rev(rev);
+        expectation.result = result.at("result").get<bool>();
+        o.expectations.push_back(expectation);
+    }
+}
+
+}  // namespace evmone::test

test/statetest/statetest.hpp

@@ -13,9 +13,6 @@ namespace json = nlohmann;
 namespace evmone::test
 {
 
-/// Translates tests fork name to EVM revision
-evmc_revision to_rev(std::string_view s);
-
 struct TestMultiTransaction : state::Transaction
 {
     struct Indexes

test/statetest/statetest_loader.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "../utils/stdx/utility.hpp"
+#include "../utils/utils.hpp"
 #include "statetest.hpp"
 #include <evmone/eof.hpp>
 #include <nlohmann/json.hpp>
@@ -211,41 +212,6 @@ state::State from_json<state::State>(const json::json& j)
     return o;
 }
 
-evmc_revision to_rev(std::string_view s)
-{
-    if (s == "Frontier")
-        return EVMC_FRONTIER;
-    if (s == "Homestead")
-        return EVMC_HOMESTEAD;
-    if (s == "EIP150")
-        return EVMC_TANGERINE_WHISTLE;
-    if (s == "EIP158")
-        return EVMC_SPURIOUS_DRAGON;
-    if (s == "Byzantium")
-        return EVMC_BYZANTIUM;
-    if (s == "Constantinople")
-        return EVMC_CONSTANTINOPLE;
-    if (s == "ConstantinopleFix")
-        return EVMC_PETERSBURG;
-    if (s == "Istanbul")
-        return EVMC_ISTANBUL;
-    if (s == "Berlin")
-        return EVMC_BERLIN;
-    if (s == "London")
-        return EVMC_LONDON;
-    if (s == "Merge")
-        return EVMC_PARIS;
-    if (s == "Merge+3855")  // PUSH0
-        return EVMC_SHANGHAI;
-    if (s == "Shanghai")
-        return EVMC_SHANGHAI;
-    if (s == "Cancun")
-        return EVMC_CANCUN;
-    if (s == "Prague")
-        return EVMC_PRAGUE;
-    throw std::invalid_argument{"unknown revision: " + std::string{s}};
-}
-
 /// Load common parts of Transaction or TestMultiTransaction.
 static void from_json_tx_common(const json::json& j, state::Transaction& o)
 {

test/t8n/t8n.cpp

@@ -5,6 +5,7 @@
 #include "../state/mpt_hash.hpp"
 #include "../state/rlp.hpp"
 #include "../statetest/statetest.hpp"
+#include "../utils/utils.hpp"
 #include <evmone/evmone.h>
 #include <evmone/version.h>
 #include <nlohmann/json.hpp>
@@ -44,7 +45,7 @@ int main(int argc, const char* argv[])
                 return 0;
             }
             if (arg == "--state.fork" && ++i < argc)
-                rev = evmone::test::to_rev(argv[i]);
+                rev = to_rev(argv[i]);
             else if (arg == "--input.alloc" && ++i < argc)
                 alloc_file = argv[i];
             else if (arg == "--input.env" && ++i < argc)

test/utils/utils.hpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 #pragma once
 
+#include <evmc/evmc.hpp>
 #include <evmc/hex.hpp>
 
 using evmc::bytes;
@@ -27,3 +28,39 @@ inline bytes operator""_hex(const char* s, size_t size)
 {
     return from_spaced_hex({s, size}).value();
 }
+
+/// Translates tests fork name to EVM revision
+inline evmc_revision to_rev(std::string_view s)
+{
+    if (s == "Frontier")
+        return EVMC_FRONTIER;
+    if (s == "Homestead")
+        return EVMC_HOMESTEAD;
+    if (s == "EIP150")
+        return EVMC_TANGERINE_WHISTLE;
+    if (s == "EIP158")
+        return EVMC_SPURIOUS_DRAGON;
+    if (s == "Byzantium")
+        return EVMC_BYZANTIUM;
+    if (s == "Constantinople")
+        return EVMC_CONSTANTINOPLE;
+    if (s == "ConstantinopleFix")
+        return EVMC_PETERSBURG;
+    if (s == "Istanbul")
+        return EVMC_ISTANBUL;
+    if (s == "Berlin")
+        return EVMC_BERLIN;
+    if (s == "London")
+        return EVMC_LONDON;
+    if (s == "Merge")
+        return EVMC_PARIS;
+    if (s == "Merge+3855")  // PUSH0
+        return EVMC_SHANGHAI;
+    if (s == "Shanghai")
+        return EVMC_SHANGHAI;
+    if (s == "Cancun")
+        return EVMC_CANCUN;
+    if (s == "Prague")
+        return EVMC_PRAGUE;
+    throw std::invalid_argument{"unknown revision: " + std::string{s}};
+}