diff --git a/circle.yml b/circle.yml index af3c6d4265..968b3bd900 100644 --- a/circle.yml +++ b/circle.yml @@ -474,13 +474,18 @@ jobs: bin/evmone-statetest ~/tests/GeneralStateTests ~/tests/LegacyTests/Constantinople/GeneralStateTests - download_execution_tests: repo: ipsilon/tests - rev: eof-rjumpv-20230803 + rev: update-tests legacy: false - run: name: "State tests (EOF)" working_directory: ~/build command: | bin/evmone-statetest ~/tests/EIPTests/StateTests/stEOF + - run: + name: "EOF validation tests" + working_directory: ~/build + command: | + bin/evmone-eoftest ~/tests/EOFTests - collect_coverage_gcc - upload_coverage: flags: statetests diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ddfb58b016..8fb8d9b30b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,10 +20,11 @@ add_subdirectory(integration) add_subdirectory(internal_benchmarks) add_subdirectory(state) add_subdirectory(statetest) +add_subdirectory(eoftest) add_subdirectory(t8n) add_subdirectory(unittests) -set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-t8n evmone-unittests) +set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-eoftest evmone-t8n evmone-unittests) if(EVMONE_FUZZING) add_subdirectory(eofparsefuzz) diff --git a/test/eoftest/.clang-tidy b/test/eoftest/.clang-tidy new file mode 100644 index 0000000000..efc628c8e9 --- /dev/null +++ b/test/eoftest/.clang-tidy @@ -0,0 +1,3 @@ +InheritParentConfig: true +Checks: > + -clang-analyzer-cplusplus.NewDeleteLeaks diff --git a/test/eoftest/CMakeLists.txt b/test/eoftest/CMakeLists.txt new file mode 100644 index 0000000000..f4147742b5 --- /dev/null +++ b/test/eoftest/CMakeLists.txt @@ -0,0 +1,15 @@ +# evmone: Fast Ethereum Virtual Machine implementation +# Copyright 2023 The evmone Authors. +# SPDX-License-Identifier: Apache-2.0 + +hunter_add_package(nlohmann_json) +find_package(nlohmann_json CONFIG REQUIRED) + +add_executable(evmone-eoftest) +target_link_libraries(evmone-eoftest PRIVATE evmone nlohmann_json::nlohmann_json GTest::gtest) +target_include_directories(evmone-eoftest PRIVATE ${evmone_private_include_dir}) +target_sources( + evmone-eoftest PRIVATE + eoftest.cpp + eoftest_runner.cpp +) diff --git a/test/eoftest/eoftest.cpp b/test/eoftest/eoftest.cpp new file mode 100644 index 0000000000..111aa38a2f --- /dev/null +++ b/test/eoftest/eoftest.cpp @@ -0,0 +1,87 @@ +// evmone: Fast Ethereum Virtual Machine implementation +// Copyright 2023 The evmone Authors. +// SPDX-License-Identifier: Apache-2.0 + + +#include "eoftest.hpp" +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace +{ + +class EOFTest : public testing::Test +{ + fs::path m_json_test_file; + +public: + explicit EOFTest(fs::path json_test_file) noexcept : m_json_test_file{std::move(json_test_file)} + {} + + void TestBody() final + { + std::ifstream f{m_json_test_file}; + evmone::test::run_eof_test(f); + } +}; + +void register_test(const std::string& suite_name, const fs::path& file) +{ + testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr, + file.string().c_str(), 0, [file]() -> testing::Test* { return new EOFTest(file); }); +} + +void register_test_files(const fs::path& root) +{ + if (is_directory(root)) + { + std::vector test_files; + std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{}, + std::back_inserter(test_files), [](const fs::directory_entry& entry) { + return entry.is_regular_file() && entry.path().extension() == ".json"; + }); + std::sort(test_files.begin(), test_files.end()); + + for (const auto& p : test_files) + register_test(fs::relative(p, root).parent_path().string(), p); + } + else // Treat as a file. + { + register_test(root.parent_path().string(), root); + } +} + +} // namespace + + +int main(int argc, char* argv[]) +{ + try + { + testing::InitGoogleTest(&argc, argv); + CLI::App app{"evmone eof test runner"}; + + std::vector paths; + app.add_option("path", paths, "Path to test file or directory") + ->required() + ->check(CLI::ExistingPath); + + CLI11_PARSE(app, argc, argv); + + for (const auto& p : paths) + { + register_test_files(p); + } + + return RUN_ALL_TESTS(); + } + catch (const std::exception& ex) + { + std::cerr << ex.what() << "\n"; + return -1; + } +} diff --git a/test/eoftest/eoftest.hpp b/test/eoftest/eoftest.hpp new file mode 100644 index 0000000000..ed4255c4d2 --- /dev/null +++ b/test/eoftest/eoftest.hpp @@ -0,0 +1,13 @@ +// evmone: Fast Ethereum Virtual Machine implementation +// Copyright 2023 The evmone Authors. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include + +namespace evmone::test +{ + +void run_eof_test(std::istream& input); + +} // namespace evmone::test diff --git a/test/eoftest/eoftest_runner.cpp b/test/eoftest/eoftest_runner.cpp new file mode 100644 index 0000000000..b2645568b6 --- /dev/null +++ b/test/eoftest/eoftest_runner.cpp @@ -0,0 +1,78 @@ +// evmone: Fast Ethereum Virtual Machine implementation +// Copyright 2023 The evmone Authors. +// SPDX-License-Identifier: Apache-2.0 + +#include "../utils/utils.hpp" +#include "eoftest.hpp" +#include +#include +#include +#include + +namespace json = nlohmann; + +namespace evmone::test +{ + +namespace +{ +struct EOFValidationTest +{ + struct Case + { + struct Expectation + { + evmc_revision rev; + bool result; + }; + std::string name; + evmc::bytes code; + std::vector expectations; + }; + std::unordered_map cases; +}; + +void from_json(const json::json& j, EOFValidationTest::Case& o) +{ + const auto op_code = evmc::from_hex(j.at("code").get()); + if (!op_code) + throw std::invalid_argument{"code is invalid hex string"}; + o.code = *op_code; + + for (const auto& [rev, result] : j.at("results").items()) + { + o.expectations.push_back({to_rev(rev), result.at("result").get()}); + } +} + +void from_json(const json::json& j, EOFValidationTest& o) +{ + if (!j.is_object() || j.empty()) + throw std::invalid_argument{"JSON test must be an object with single key of the test name"}; + + const auto& j_t = *j.begin(); // Content is in a dict with the test name. + + for (const auto& [name, test] : j_t.at("vectors").items()) + { + o.cases.emplace(name, test.get()); + } +} + +} // namespace + +void run_eof_test(std::istream& input) +{ + const auto test = json::json::parse(input).get(); + for (const auto& [name, cases] : test.cases) + { + for (const auto& expectation : cases.expectations) + { + const auto result = evmone::validate_eof(expectation.rev, cases.code); + const bool b_result = (result == EOFValidationError::success); + EXPECT_EQ(b_result, expectation.result) + << name << " " << expectation.rev << " " << hex(cases.code); + } + } +} + +} // namespace evmone::test \ No newline at end of file diff --git a/test/statetest/statetest.hpp b/test/statetest/statetest.hpp index a4cf8f54d4..645657ad98 100644 --- a/test/statetest/statetest.hpp +++ b/test/statetest/statetest.hpp @@ -13,9 +13,6 @@ namespace json = nlohmann; namespace evmone::test { -/// Translates tests fork name to EVM revision -evmc_revision to_rev(std::string_view s); - struct TestMultiTransaction : state::Transaction { struct Indexes diff --git a/test/statetest/statetest_loader.cpp b/test/statetest/statetest_loader.cpp index f46f36e761..ab99f59f5c 100644 --- a/test/statetest/statetest_loader.cpp +++ b/test/statetest/statetest_loader.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "../utils/stdx/utility.hpp" +#include "../utils/utils.hpp" #include "statetest.hpp" #include #include @@ -211,41 +212,6 @@ state::State from_json(const json::json& j) return o; } -evmc_revision to_rev(std::string_view s) -{ - if (s == "Frontier") - return EVMC_FRONTIER; - if (s == "Homestead") - return EVMC_HOMESTEAD; - if (s == "EIP150") - return EVMC_TANGERINE_WHISTLE; - if (s == "EIP158") - return EVMC_SPURIOUS_DRAGON; - if (s == "Byzantium") - return EVMC_BYZANTIUM; - if (s == "Constantinople") - return EVMC_CONSTANTINOPLE; - if (s == "ConstantinopleFix") - return EVMC_PETERSBURG; - if (s == "Istanbul") - return EVMC_ISTANBUL; - if (s == "Berlin") - return EVMC_BERLIN; - if (s == "London") - return EVMC_LONDON; - if (s == "Merge") - return EVMC_PARIS; - if (s == "Merge+3855") // PUSH0 - return EVMC_SHANGHAI; - if (s == "Shanghai") - return EVMC_SHANGHAI; - if (s == "Cancun") - return EVMC_CANCUN; - if (s == "Prague") - return EVMC_PRAGUE; - throw std::invalid_argument{"unknown revision: " + std::string{s}}; -} - /// Load common parts of Transaction or TestMultiTransaction. static void from_json_tx_common(const json::json& j, state::Transaction& o) { diff --git a/test/t8n/t8n.cpp b/test/t8n/t8n.cpp index c96dbecf47..87887974ff 100644 --- a/test/t8n/t8n.cpp +++ b/test/t8n/t8n.cpp @@ -6,6 +6,7 @@ #include "../state/mpt_hash.hpp" #include "../state/rlp.hpp" #include "../statetest/statetest.hpp" +#include "../utils/utils.hpp" #include #include #include @@ -45,7 +46,7 @@ int main(int argc, const char* argv[]) return 0; } if (arg == "--state.fork" && ++i < argc) - rev = evmone::test::to_rev(argv[i]); + rev = to_rev(argv[i]); else if (arg == "--input.alloc" && ++i < argc) alloc_file = argv[i]; else if (arg == "--input.env" && ++i < argc) diff --git a/test/utils/utils.hpp b/test/utils/utils.hpp index 0c5ff7c45c..f0b0247052 100644 --- a/test/utils/utils.hpp +++ b/test/utils/utils.hpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include using evmc::bytes; @@ -27,3 +28,39 @@ inline bytes operator""_hex(const char* s, size_t size) { return from_spaced_hex({s, size}).value(); } + +/// Translates tests fork name to EVM revision +inline evmc_revision to_rev(std::string_view s) +{ + if (s == "Frontier") + return EVMC_FRONTIER; + if (s == "Homestead") + return EVMC_HOMESTEAD; + if (s == "EIP150") + return EVMC_TANGERINE_WHISTLE; + if (s == "EIP158") + return EVMC_SPURIOUS_DRAGON; + if (s == "Byzantium") + return EVMC_BYZANTIUM; + if (s == "Constantinople") + return EVMC_CONSTANTINOPLE; + if (s == "ConstantinopleFix") + return EVMC_PETERSBURG; + if (s == "Istanbul") + return EVMC_ISTANBUL; + if (s == "Berlin") + return EVMC_BERLIN; + if (s == "London") + return EVMC_LONDON; + if (s == "Merge") + return EVMC_PARIS; + if (s == "Merge+3855") // PUSH0 + return EVMC_SHANGHAI; + if (s == "Shanghai") + return EVMC_SHANGHAI; + if (s == "Cancun") + return EVMC_CANCUN; + if (s == "Prague") + return EVMC_PRAGUE; + throw std::invalid_argument{"unknown revision: " + std::string{s}}; +}