Project benchmarks

.gitignore

@@ -8,3 +8,4 @@ doc
 docs
 /test/tmp
 save.benchee
+/.elixir_ls/

README.md

@@ -212,6 +212,7 @@ The available options are the following (also documented in [hexdocs](https://he
 * `print` - a map or keyword list from atoms to `true` or `false` to configure if the output identified by the atom will be printed during the standard benchee benchmarking process. All options are enabled by default (true). Options are:
   * `:benchmarking`  - print when Benchee starts benchmarking a new job (`Benchmarking name ...`)
   * `:configuration` - a summary of configured benchmarking options including estimated total run time is printed before benchmarking starts
+  * `:system` - a description of a system benchmark is being run on
   * `:fast_warning` - warnings are displayed if functions are executed too fast leading to inaccurate measures
 * `:unit_scaling` - the strategy for choosing a unit for durations,
   counts & memory measurements. May or may not be implemented by a given formatter (The console formatter implements it).

example/project_benchmark.exs

@@ -0,0 +1,87 @@
+defmodule Clickhouse.Benchmark.Performance do
+  use Benchee.Benchmark
+
+  # Since it's a module, any other constructs may be used,
+  # such as defs, defmacro, etc.
+  def map_fun(i) do
+    [i]
+  end
+
+  # Global benchmark setup. Optional.
+  #
+  # Called before any benchmark in the module.
+  #
+  # If defined, must return {:ok, state} for benchmarks to run
+  before_all do
+    {:ok, nil}
+  end
+
+  # Global benchmark teardown. Optional.
+  #
+  # Called after all benchmarks have finished
+  # (and their possible local teardowns had been called).
+  #
+  # Can return anything.
+  after_all do
+    :anything
+  end
+
+  # Benchmarks. Module may have many of them.
+  benchmark "Flattening list from map", # Name. Mandatory.
+  warmup: 0, time: 1          # Opts (see Benchee.run). Optional.
+    do                                  # Do block. Mandatory.
+
+    # Benchmark setup. Optional
+    #
+    # If global setup is defined,
+    # implicit variable "state" is bound to it's result
+    #
+    # If defined, must return {:ok, state} for benchmarks to run
+    before_benchmark do
+      {:ok, fn x -> [x] end}
+    end
+
+    # Benchmark teardown. Optional
+    #
+    # Implicit variable: state, as it is returned from local setup
+    # (or from global setup, if no local setup is defined)
+    #
+    # Can return anything
+    after_benchmark do
+      :anything
+    end
+
+    # Inputs: the same as passing inputs via :input option
+    #
+    # Accepts either an expression or a do block as a 2nd argument
+    input "Small", Enum.to_list(1..100)
+    input "Medium" do
+      n = 10_000
+      Enum.to_list(1..n)
+    end
+    input "Bigger", Enum.to_list(1..100_000)
+
+
+    # Benchmark scenarios
+    #
+    # Each scenario has an implicit variables:
+    # - state: state returned from local or global setup
+    # And if any input is passed (either with option or as input directive):
+    # - input: data for benchmark
+    scenario "Enum.flat_map", # Name. Mandatory
+      before_scenario:
+        fn i ->               # Scenario options, e.g. local hooks. Optional.
+          IO.inspect(length(i), label: "Input length");
+          i
+        end
+      do                      # Do block. Mandatory
+        map_fun = state
+        Enum.flat_map(input, map_fun)
+      end
+
+    scenario "Enum.map |> List.flatten" do
+      map_fun = state
+      input |> Enum.map(map_fun) |> List.flatten()
+    end
+  end
+end

lib/benchee/benchmark.ex

@@ -98,9 +98,22 @@ defmodule Benchee.Benchmark do
         printer \\ Printer,
         runner \\ Runner
       ) do
+    printer.system_information(suite)
     printer.configuration_information(suite)
     scenario_context = %ScenarioContext{config: config, printer: printer}
     scenarios = runner.run_scenarios(scenarios, scenario_context)
     %Suite{suite | scenarios: scenarios}
   end
+
+  # delegate use to Benchee.Project.Benchmark.__using__/1
+  defmacro __using__(opts) do
+    require Benchee.Project.Benchmark
+
+    quoted =
+      quote do
+        use Benchee.Project.Benchmark, unquote(opts)
+      end
+
+    Macro.expand(quoted, __ENV__)
+  end
 end

lib/benchee/configuration.ex

@@ -22,8 +22,9 @@ defmodule Benchee.Configuration do
             percentiles: [50, 99],
             print: %{
               benchmarking: true,
+              fast_warning: true,
               configuration: true,
-              fast_warning: true
+              system: true
             },
             inputs: nil,
             save: false,
@@ -178,7 +179,8 @@ defmodule Benchee.Configuration do
             print: %{
               benchmarking: true,
               fast_warning: true,
-              configuration: true
+              configuration: true,
+              system: true
             },
             percentiles: [50, 99],
             unit_scaling: :best,
@@ -206,7 +208,8 @@ defmodule Benchee.Configuration do
             print: %{
               benchmarking: true,
               fast_warning: true,
-              configuration: true
+              configuration: true,
+              system: true
             },
             percentiles: [50, 99],
             unit_scaling: :best,
@@ -234,7 +237,8 @@ defmodule Benchee.Configuration do
             print: %{
               benchmarking: true,
               fast_warning: true,
-              configuration: true
+              configuration: true,
+              system: true
             },
             percentiles: [50, 99],
             unit_scaling: :best,
@@ -269,7 +273,8 @@ defmodule Benchee.Configuration do
             print: %{
               benchmarking: true,
               fast_warning: false,
-              configuration: true
+              configuration: true,
+              system: true
             },
             percentiles: [50, 99],
             unit_scaling: :smallest,

lib/benchee/output/benchmark_printer.ex

@@ -15,19 +15,17 @@ defmodule Benchee.Output.BenchmarkPrinter do
   end
 
   @doc """
-  Prints general information such as system information and estimated
-  benchmarking time.
+  Prints general information about the system such as operating system.
   """
-  def configuration_information(%{configuration: %{print: %{configuration: false}}}) do
+  def system_information(%{configuration: %{print: %{system: false}}}) do
     nil
   end
 
-  def configuration_information(%{scenarios: scenarios, system: sys, configuration: config}) do
-    system_information(sys)
-    suite_information(scenarios, config)
+  def system_information(%{system: sys}) do
+    print_system_information(sys)
   end
 
-  defp system_information(%{
+  defp print_system_information(%{
          erlang: erlang_version,
          elixir: elixir_version,
          os: os,
@@ -45,7 +43,18 @@ defmodule Benchee.Output.BenchmarkPrinter do
     """)
   end
 
-  defp suite_information(scenarios, %{
+  @doc """
+  Prints general benchmark information such as estimated benchmarking time.
+  """
+  def configuration_information(%{configuration: %{print: %{configuration: false}}}) do
+    nil
+  end
+
+  def configuration_information(%{scenarios: scenarios, configuration: config}) do
+    print_suite_information(scenarios, config)
+  end
+
+  defp print_suite_information(scenarios, %{
          parallel: parallel,
          time: time,
          warmup: warmup,