From 0b93b42ffbf8a55effff146053522929147cad1e Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Mon, 18 Dec 2023 10:02:12 +0100 Subject: [PATCH 1/2] Introduce `BenchmarkConfig` to selectively use config in benchmark Fixes #412 - but basically the `Config` holds `inputs` and that can be a hell loat of data to send along with every benchmark. In a benchmark with 16 scenarios and huge inputs we now save ourselves copying that potentially huge data structure a whopping 16 times to our benchmarking processes for no reason whatsoever (the scenarios already hold the `input`). --- lib/benchee/benchmark.ex | 11 ++++- lib/benchee/benchmark/benchmark_config.ex | 53 +++++++++++++++++++++++ lib/benchee/benchmark/runner.ex | 10 ++--- lib/benchee/benchmark/scenario_context.ex | 2 +- lib/benchee/configuration.ex | 2 +- lib/benchee/profile.ex | 3 +- test/benchee/benchmark_test.exs | 5 ++- 7 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 lib/benchee/benchmark/benchmark_config.ex diff --git a/lib/benchee/benchmark.ex b/lib/benchee/benchmark.ex index 5e59527a..2b843e29 100644 --- a/lib/benchee/benchmark.ex +++ b/lib/benchee/benchmark.ex @@ -4,7 +4,7 @@ defmodule Benchee.Benchmark do Exposes `benchmark/4` and `collect/3` functions. """ - alias Benchee.Benchmark.{Runner, ScenarioContext} + alias Benchee.Benchmark.{BenchmarkConfig, Runner, ScenarioContext} alias Benchee.Output.BenchmarkPrinter, as: Printer alias Benchee.Scenario alias Benchee.Suite @@ -118,7 +118,14 @@ defmodule Benchee.Benchmark do runner \\ Runner ) do printer.configuration_information(suite) - scenario_context = %ScenarioContext{config: config, printer: printer, system: system} + benchmark_config = BenchmarkConfig.from(config) + + scenario_context = %ScenarioContext{ + config: benchmark_config, + printer: printer, + system: system + } + scenarios = runner.run_scenarios(scenarios, scenario_context) %Suite{suite | scenarios: scenarios} end diff --git a/lib/benchee/benchmark/benchmark_config.ex b/lib/benchee/benchmark/benchmark_config.ex new file mode 100644 index 00000000..179bb651 --- /dev/null +++ b/lib/benchee/benchmark/benchmark_config.ex @@ -0,0 +1,53 @@ +defmodule Benchee.Benchmark.BenchmarkConfig do + @moduledoc """ + Benchmark Configuration, practically a sub set of `Benchee.Configuration` + + `Benchee.Configuration` holds too much data that we don't want to send into the benchmarking + processes - inputs being potentially huge. Hence, we take the sub set the benchmarks need and + put it in here. Since this is a benchmarking library, to no one's surprise these are a lot of + them. + See: https://github.com/bencheeorg/benchee/issues/412 + """ + + alias Benchee.Benchmark.Hooks + + @keys [ + :warmup, + :time, + :memory_time, + :reduction_time, + :pre_check, + :measure_function_call_overhead, + :before_each, + :after_each, + :before_scenario, + :after_scenario, + :parallel, + :print + ] + + defstruct @keys + + @type t :: %__MODULE__{ + time: number, + warmup: number, + memory_time: number, + reduction_time: number, + pre_check: boolean, + measure_function_call_overhead: boolean, + print: map, + before_each: Hooks.hook_function() | nil, + after_each: Hooks.hook_function() | nil, + before_scenario: Hooks.hook_function() | nil, + after_scenario: Hooks.hook_function() | nil, + measure_function_call_overhead: boolean, + parallel: pos_integer() + } + + alias Benchee.Configuration + + def from(config = %Configuration{}) do + values = Map.take(config, @keys) + struct!(__MODULE__, values) + end +end diff --git a/lib/benchee/benchmark/runner.ex b/lib/benchee/benchmark/runner.ex index c0467270..488bc7d5 100644 --- a/lib/benchee/benchmark/runner.ex +++ b/lib/benchee/benchmark/runner.ex @@ -6,7 +6,7 @@ defmodule Benchee.Benchmark.Runner do # This module actually runs our benchmark scenarios, adding information about # run time and memory usage to each scenario. - alias Benchee.{Benchmark, Configuration, Scenario, Utility.Parallel} + alias Benchee.{Benchmark, Scenario, Utility.Parallel} alias Benchmark.{ Collect, @@ -121,7 +121,7 @@ defmodule Benchee.Benchmark.Runner do defp run_warmup( scenario, scenario_context = %ScenarioContext{ - config: %Configuration{warmup: warmup} + config: %{warmup: warmup} } ) do measure_runtimes(scenario, scenario_context, warmup, false) @@ -130,7 +130,7 @@ defmodule Benchee.Benchmark.Runner do defp run_runtime_benchmark( scenario, scenario_context = %ScenarioContext{ - config: %Configuration{ + config: %{ time: run_time, print: %{fast_warning: fast_warning} } @@ -175,7 +175,7 @@ defmodule Benchee.Benchmark.Runner do defp run_reductions_benchmark( scenario, scenario_context = %ScenarioContext{ - config: %Configuration{ + config: %{ reduction_time: reduction_time } } @@ -198,7 +198,7 @@ defmodule Benchee.Benchmark.Runner do defp run_memory_benchmark( scenario, scenario_context = %ScenarioContext{ - config: %Configuration{ + config: %{ memory_time: memory_time } } diff --git a/lib/benchee/benchmark/scenario_context.ex b/lib/benchee/benchmark/scenario_context.ex index d06720da..a4a156c9 100644 --- a/lib/benchee/benchmark/scenario_context.ex +++ b/lib/benchee/benchmark/scenario_context.ex @@ -22,7 +22,7 @@ defmodule Benchee.Benchmark.ScenarioContext do ] @type t :: %__MODULE__{ - config: Benchee.Configuration.t(), + config: Benchee.Benchmark.BenchmarkConfig.t(), printer: module, current_time: pos_integer | nil, end_time: pos_integer | nil, diff --git a/lib/benchee/configuration.ex b/lib/benchee/configuration.ex index 850b1d11..250d15c0 100644 --- a/lib/benchee/configuration.ex +++ b/lib/benchee/configuration.ex @@ -142,7 +142,7 @@ defmodule Benchee.Configuration do a map or struct at this point for easier handling in Benchee. """ @type t :: %__MODULE__{ - parallel: integer, + parallel: pos_integer, time: number, warmup: number, memory_time: number, diff --git a/lib/benchee/profile.ex b/lib/benchee/profile.ex index f7e6bf7a..ba2d844f 100644 --- a/lib/benchee/profile.ex +++ b/lib/benchee/profile.ex @@ -1,4 +1,5 @@ defmodule Benchee.Profile do + alias Benchee.Benchmark.BenchmarkConfig alias Benchee.Benchmark.Collect alias Benchee.Benchmark.RunOnce alias Benchee.Benchmark.ScenarioContext @@ -104,7 +105,7 @@ defmodule Benchee.Profile do RunOnce.run( scenario, - %ScenarioContext{config: config}, + %ScenarioContext{config: BenchmarkConfig.from(config)}, {Collect.Profile, [profiler_module: profiler_module, profiler_opts: profiler_opts]} ) end diff --git a/test/benchee/benchmark_test.exs b/test/benchee/benchmark_test.exs index 3958534a..63c12e78 100644 --- a/test/benchee/benchmark_test.exs +++ b/test/benchee/benchmark_test.exs @@ -8,6 +8,8 @@ defmodule Benchee.BenchmarkTest do Suite } + alias Benchee.Benchmark.BenchmarkConfig + alias Benchee.Benchmark.ScenarioContext alias Benchee.Test.FakeBenchmarkPrinter, as: TestPrinter alias Benchee.Test.FakeBenchmarkRunner, as: TestRunner @@ -104,8 +106,9 @@ defmodule Benchee.BenchmarkTest do test "sends the correct data to the benchmark runner" do scenarios = [%Scenario{job_name: "job_one"}] config = %Configuration{} + benchmark_config = BenchmarkConfig.from(config) suite = %Suite{scenarios: scenarios, configuration: config} - scenario_context = %ScenarioContext{config: config, printer: TestPrinter} + scenario_context = %ScenarioContext{config: benchmark_config, printer: TestPrinter} Benchmark.collect(suite, TestPrinter, TestRunner) From 3d862b69337be820495fac51965ea662c3a9c889 Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Mon, 18 Dec 2023 10:14:51 +0100 Subject: [PATCH 2/2] Move moduledoc to top of module --- lib/benchee/profile.ex | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/benchee/profile.ex b/lib/benchee/profile.ex index ba2d844f..e85e4122 100644 --- a/lib/benchee/profile.ex +++ b/lib/benchee/profile.ex @@ -1,20 +1,4 @@ defmodule Benchee.Profile do - alias Benchee.Benchmark.BenchmarkConfig - alias Benchee.Benchmark.Collect - alias Benchee.Benchmark.RunOnce - alias Benchee.Benchmark.ScenarioContext - alias Benchee.Output.ProfilePrinter, as: Printer - alias Benchee.Suite - - @default_profiler :eprof - @builtin_profilers [:cprof, :eprof, :fprof] - # we run the function a bunch already, no need for further warmup - @default_profiler_opts [warmup: false] - - defmodule Benchee.UnknownProfilerError do - defexception message: "error" - end - @moduledoc """ Profiles each scenario after benchmarking them if the `profile_after` option is either set to: * `true`, @@ -32,6 +16,22 @@ defmodule Benchee.Profile do *excluding* the time of called functions. """ + alias Benchee.Benchmark.BenchmarkConfig + alias Benchee.Benchmark.Collect + alias Benchee.Benchmark.RunOnce + alias Benchee.Benchmark.ScenarioContext + alias Benchee.Output.ProfilePrinter, as: Printer + alias Benchee.Suite + + @default_profiler :eprof + @builtin_profilers [:cprof, :eprof, :fprof] + # we run the function a bunch already, no need for further warmup + @default_profiler_opts [warmup: false] + + defmodule Benchee.UnknownProfilerError do + defexception message: "error" + end + @doc """ Returns the atom corresponding to the default profiler. """