Merge pull request #343 from ethereum/bench_exec_only

bench: Add separate benchmarks for execution only

lib/evmone/baseline.hpp

@@ -31,7 +31,8 @@ evmc_result execute(evmc_vm* vm, const evmc_host_interface* host, evmc_host_cont
     evmc_revision rev, const evmc_message* msg, const uint8_t* code, size_t code_size) noexcept;
 
 /// Executes in Baseline interpreter on the given external and initialized state.
-evmc_result execute(const VM&, ExecutionState& state, const CodeAnalysis& analysis) noexcept;
+EVMC_EXPORT evmc_result execute(
+    const VM&, ExecutionState& state, const CodeAnalysis& analysis) noexcept;
 
 }  // namespace baseline
 }  // namespace evmone

lib/evmone/execution.hpp

@@ -4,14 +4,16 @@
 #pragma once
 
 #include <evmc/evmc.h>
+#include <evmc/utils.h>
 
 namespace evmone
 {
 struct AdvancedExecutionState;
 struct AdvancedCodeAnalysis;
 
 /// Execute the already analyzed code using the provided execution state.
-evmc_result execute(AdvancedExecutionState& state, const AdvancedCodeAnalysis& analysis) noexcept;
+EVMC_EXPORT evmc_result execute(
+    AdvancedExecutionState& state, const AdvancedCodeAnalysis& analysis) noexcept;
 
 /// EVMC-compatible execute() function.
 evmc_result execute(evmc_vm* vm, const evmc_host_interface* host, evmc_host_context* ctx,

test/bench/CMakeLists.txt

@@ -47,12 +47,12 @@ set(PREFIX evmone/bench)
 
 # Check if DIR argument works.
 add_test(NAME ${PREFIX}/dir COMMAND evmone-bench ${CMAKE_CURRENT_SOURCE_DIR}/../benchmarks --benchmark_list_tests)
-set_tests_properties(${PREFIX}/dir PROPERTIES PASS_REGULAR_EXPRESSION "execute/synth")
+set_tests_properties(${PREFIX}/dir PROPERTIES PASS_REGULAR_EXPRESSION "total/synth")
 
 # Omitting DIR is fine.
 add_test(NAME ${PREFIX}/no_dir COMMAND evmone-bench --benchmark_list_tests)
-set_tests_properties(${PREFIX}/no_dir PROPERTIES PASS_REGULAR_EXPRESSION "execute/synth")
+set_tests_properties(${PREFIX}/no_dir PROPERTIES PASS_REGULAR_EXPRESSION "total/synth")
 
 # Empty DIR name should list only built-in benchmarks
 add_test(NAME ${PREFIX}/dirname_empty COMMAND evmone-bench "" --benchmark_list_tests)
-set_tests_properties(${PREFIX}/dirname_empty PROPERTIES PASS_REGULAR_EXPRESSION "execute/synth")
+set_tests_properties(${PREFIX}/dirname_empty PROPERTIES PASS_REGULAR_EXPRESSION "total/synth")

test/bench/bench.cpp

@@ -159,31 +159,56 @@ std::vector<BenchmarkCase> load_benchmarks_from_dir(
 
 void register_benchmarks(const std::vector<BenchmarkCase>& benchmark_cases)
 {
+    evmc::VM* advanced_vm = nullptr;
+    evmc::VM* baseline_vm = nullptr;
+    if (const auto it = registered_vms.find("advanced"); it != registered_vms.end())
+        advanced_vm = &it->second;
+    if (const auto it = registered_vms.find("baseline"); it != registered_vms.end())
+        baseline_vm = &it->second;
+
     for (const auto& b : benchmark_cases)
     {
-        if (registered_vms.count("advanced"))
+        if (advanced_vm)
         {
             RegisterBenchmark(("advanced/analyse/" + b.name).c_str(), [&b](State& state) {
-                analyse(state, default_revision, b.code);
+                bench_analyse<AdvancedCodeAnalysis, advanced_analyse>(
+                    state, default_revision, b.code);
             })->Unit(kMicrosecond);
         }
 
-        if (registered_vms.count("baseline"))
+        if (baseline_vm)
         {
             RegisterBenchmark(("baseline/analyse/" + b.name).c_str(), [&b](State& state) {
-                baseline_analyze(state, b.code);
+                bench_analyse<baseline::CodeAnalysis, baseline_analyse>(
+                    state, default_revision, b.code);
             })->Unit(kMicrosecond);
         }
 
         for (const auto& input : b.inputs)
         {
-            const auto case_name =
-                "/execute/" + b.name + (!input.name.empty() ? '/' + input.name : "");
+            const auto case_name = b.name + (!input.name.empty() ? '/' + input.name : "");
+
+            if (advanced_vm)
+            {
+                const auto name = "advanced/execute/" + case_name;
+                RegisterBenchmark(name.c_str(), [&vm = *advanced_vm, &b, &input](State& state) {
+                    bench_advanced_execute(state, vm, b.code, input.input, input.expected_output);
+                })->Unit(kMicrosecond);
+            }
+
+            if (baseline_vm)
+            {
+                const auto name = "baseline/execute/" + case_name;
+                RegisterBenchmark(name.c_str(), [&vm = *baseline_vm, &b, &input](State& state) {
+                    bench_baseline_execute(state, vm, b.code, input.input, input.expected_output);
+                })->Unit(kMicrosecond);
+            }
+
             for (auto& [vm_name, vm] : registered_vms)
             {
-                const auto name = std::string{vm_name} + case_name;
+                const auto name = std::string{vm_name} + "/total/" + case_name;
                 RegisterBenchmark(name.c_str(), [&vm = vm, &b, &input](State& state) {
-                    execute(state, vm, b.code, input.input, input.expected_output);
+                    bench_evmc_execute(state, vm, b.code, input.input, input.expected_output);
                 })->Unit(kMicrosecond);
             }
         }

test/bench/helpers.hpp

@@ -9,6 +9,8 @@
 #include <evmc/mocked_host.hpp>
 #include <evmone/analysis.hpp>
 #include <evmone/baseline.hpp>
+#include <evmone/execution.hpp>
+#include <evmone/vm.hpp>
 
 namespace evmone::test
 {
@@ -17,27 +19,70 @@ extern std::map<std::string_view, evmc::VM> registered_vms;
 constexpr auto default_revision = EVMC_ISTANBUL;
 constexpr auto default_gas_limit = std::numeric_limits<int64_t>::max();
 
-inline void analyse(benchmark::State& state, evmc_revision rev, bytes_view code) noexcept
+
+template <typename ExecutionStateT, typename AnalysisT>
+using ExecuteFn = evmc::result(evmc::VM& vm, ExecutionStateT& exec_state, const AnalysisT&,
+    const evmc_message&, evmc_revision, evmc::Host&, bytes_view);
+
+template <typename AnalysisT>
+using AnalyseFn = AnalysisT(evmc_revision, bytes_view);
+
+
+struct FakeExecutionState
+{};
+
+struct FakeCodeAnalysis
+{};
+
+inline AdvancedCodeAnalysis advanced_analyse(evmc_revision rev, bytes_view code)
 {
-    auto bytes_analysed = uint64_t{0};
-    for (auto _ : state)
-    {
-        auto r = evmone::analyze(rev, code.data(), code.size());
-        benchmark::DoNotOptimize(&r);
-        bytes_analysed += code.size();
-    }
+    // TODO: Change analyze() signature.
+    return analyze(rev, code.data(), code.size());
+}
 
-    using benchmark::Counter;
-    state.counters["size"] = Counter(static_cast<double>(code.size()));
-    state.counters["rate"] = Counter(static_cast<double>(bytes_analysed), Counter::kIsRate);
+inline baseline::CodeAnalysis baseline_analyse(evmc_revision /*rev*/, bytes_view code)
+{
+    return baseline::analyze(code.data(), code.size());
+}
+
+inline FakeCodeAnalysis evmc_analyse(evmc_revision /*rev*/, bytes_view /*code*/)
+{
+    return {};
+}
+
+
+inline evmc::result advanced_execute(evmc::VM& /*vm*/, AdvancedExecutionState& exec_state,
+    const AdvancedCodeAnalysis& analysis, const evmc_message& msg, evmc_revision rev,
+    evmc::Host& host, bytes_view code)
+{
+    exec_state.reset(msg, rev, host.get_interface(), host.to_context(), code.data(), code.size());
+    return evmc::result{execute(exec_state, analysis)};
 }
 
-inline void baseline_analyze(benchmark::State& state, bytes_view code) noexcept
+inline evmc::result baseline_execute(evmc::VM& c_vm, ExecutionState& exec_state,
+    const baseline::CodeAnalysis& analysis, const evmc_message& msg, evmc_revision rev,
+    evmc::Host& host, bytes_view code)
+{
+    const auto& vm = *static_cast<evmone::VM*>(c_vm.get_raw_pointer());
+    exec_state.reset(msg, rev, host.get_interface(), host.to_context(), code.data(), code.size());
+    return evmc::result{baseline::execute(vm, exec_state, analysis)};
+}
+
+inline evmc::result evmc_execute(evmc::VM& vm, FakeExecutionState& /*exec_state*/,
+    const FakeCodeAnalysis& /*analysis*/, const evmc_message& msg, evmc_revision rev,
+    evmc::Host& host, bytes_view code) noexcept
+{
+    return vm.execute(host, rev, msg, code.data(), code.size());
+}
+
+
+template <typename AnalysisT, AnalyseFn<AnalysisT> analyse_fn>
+inline void bench_analyse(benchmark::State& state, evmc_revision rev, bytes_view code) noexcept
 {
     auto bytes_analysed = uint64_t{0};
     for (auto _ : state)
     {
-        auto r = evmone::baseline::analyze(code.data(), code.size());
+        auto r = analyse_fn(rev, code);
         benchmark::DoNotOptimize(&r);
         bytes_analysed += code.size();
     }
@@ -47,26 +92,27 @@ inline void baseline_analyze(benchmark::State& state, bytes_view code) noexcept
     state.counters["rate"] = Counter(static_cast<double>(bytes_analysed), Counter::kIsRate);
 }
 
-inline evmc::result execute(evmc::VM& vm, evmc::Host& host, evmc_revision rev, int64_t gas_limit,
-    bytes_view code, bytes_view input) noexcept
-{
-    auto msg = evmc_message{};
-    msg.gas = gas_limit;
-    msg.input_data = input.data();
-    msg.input_size = input.size();
-    return vm.execute(host, rev, msg, code.data(), code.size());
-}
 
-inline void execute(benchmark::State& state, evmc::VM& vm, bytes_view code, bytes_view input = {},
-    bytes_view expected_output = {}) noexcept
+template <typename ExecutionStateT, typename AnalysisT,
+    ExecuteFn<ExecutionStateT, AnalysisT> execute_fn, AnalyseFn<AnalysisT> analyse_fn>
+inline void bench_execute(benchmark::State& state, evmc::VM& vm, bytes_view code, bytes_view input,
+    bytes_view expected_output) noexcept
 {
     constexpr auto rev = default_revision;
     constexpr auto gas_limit = default_gas_limit;
 
+    const auto analysis = analyse_fn(rev, code);
     evmc::MockedHost host;
+    ExecutionStateT exec_state;
+    evmc_message msg{};
+    msg.kind = EVMC_CALL;
+    msg.gas = gas_limit;
+    msg.input_data = input.data();
+    msg.input_size = input.size();
+
 
     {  // Test run.
-        const auto r = execute(vm, host, rev, gas_limit, code, input);
+        const auto r = execute_fn(vm, exec_state, analysis, msg, rev, host, code);
         if (r.status_code != EVMC_SUCCESS)
         {
             state.SkipWithError(("failure: " + std::to_string(r.status_code)).c_str());
@@ -89,7 +135,7 @@ inline void execute(benchmark::State& state, evmc::VM& vm, bytes_view code, byte
     auto iteration_gas_used = int64_t{0};
     for (auto _ : state)
     {
-        auto r = execute(vm, host, rev, gas_limit, code, input);
+        const auto r = execute_fn(vm, exec_state, analysis, msg, rev, host, code);
         iteration_gas_used = gas_limit - r.gas_left;
         total_gas_used += iteration_gas_used;
     }
@@ -98,4 +144,19 @@ inline void execute(benchmark::State& state, evmc::VM& vm, bytes_view code, byte
     state.counters["gas_used"] = Counter(static_cast<double>(iteration_gas_used));
     state.counters["gas_rate"] = Counter(static_cast<double>(total_gas_used), Counter::kIsRate);
 }
+
+
+constexpr auto bench_advanced_execute =
+    bench_execute<AdvancedExecutionState, AdvancedCodeAnalysis, advanced_execute, advanced_analyse>;
+
+constexpr auto bench_baseline_execute =
+    bench_execute<ExecutionState, baseline::CodeAnalysis, baseline_execute, baseline_analyse>;
+
+inline void bench_evmc_execute(benchmark::State& state, evmc::VM& vm, bytes_view code,
+    bytes_view input = {}, bytes_view expected_output = {})
+{
+    bench_execute<FakeExecutionState, FakeCodeAnalysis, evmc_execute, evmc_analyse>(
+        state, vm, code, input, expected_output);
+}
+
 }  // namespace evmone::test

test/bench/synthetic_benchmarks.cpp

@@ -257,19 +257,19 @@ void register_synthetic_benchmarks()
 
     for (auto& [vm_name, vm] : registered_vms)
     {
-        RegisterBenchmark((std::string{vm_name} + "/execute/synth/loop_v1").c_str(),
-            [&vm = vm](State& state) { execute(state, vm, generate_loop_v1({})); });
-        RegisterBenchmark((std::string{vm_name} + "/execute/synth/loop_v2").c_str(),
-            [&vm = vm](State& state) { execute(state, vm, generate_loop_v2({})); });
+        RegisterBenchmark((std::string{vm_name} + "/total/synth/loop_v1").c_str(),
+            [&vm = vm](State& state) { bench_evmc_execute(state, vm, generate_loop_v1({})); });
+        RegisterBenchmark((std::string{vm_name} + "/total/synth/loop_v2").c_str(),
+            [&vm = vm](State& state) { bench_evmc_execute(state, vm, generate_loop_v2({})); });
     }
 
     for (const auto params : params_list)
     {
         for (auto& [vm_name, vm] : registered_vms)
         {
-            RegisterBenchmark(
-                (std::string{vm_name} + "/execute/synth/" + to_string(params)).c_str(),
-                [&vm = vm, params](State& state) { execute(state, vm, generate_code(params)); })
+            RegisterBenchmark((std::string{vm_name} + "/total/synth/" + to_string(params)).c_str(),
+                [&vm = vm, params](
+                    State& state) { bench_evmc_execute(state, vm, generate_code(params)); })
                 ->Unit(kMicrosecond);
         }
     }