From 6eadf7ad40fcb567b6e26ceab794d068b2e0bac8 Mon Sep 17 00:00:00 2001 From: Marcus Kruse Date: Sun, 17 Dec 2023 09:09:43 +0100 Subject: [PATCH 1/5] Add updates for benchee 1.3 --- CHANGELOG.md | 6 +- examples/bench/basic/README.md | 120 ++---------- examples/bench/no_input/README.md | 208 +++++++++++++++++++++ examples/bench/no_input/run.exs | 21 +++ lib/benchee/formatters/markdown/helpers.ex | 4 + mix.exs | 2 +- 6 files changed, 258 insertions(+), 103 deletions(-) create mode 100644 examples/bench/no_input/README.md create mode 100644 examples/bench/no_input/run.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6224f49..d623ae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog -## 0.3.1 - 2022-11-18 +## 0.3.3 - 2023-12-17 + +Changes for `benchee` 1.3. + +## 0.3.2 - 2022-11-18 Create dirs for output files. diff --git a/examples/bench/basic/README.md b/examples/bench/basic/README.md index 574260e..598ce63 100644 --- a/examples/bench/basic/README.md +++ b/examples/bench/basic/README.md @@ -1,6 +1,6 @@ Benchmark Example -Benchmark run from 2022-11-18 15:22:20.915093Z UTC +Benchmark run from 2023-12-17 06:58:37.321435Z UTC ## System @@ -21,10 +21,10 @@ Benchmark suite executing on the following system: 16 GB Elixir Version - 1.14.1 + 1.15.7 Erlang Version - 25.1.2 + 26.2 @@ -49,88 +49,6 @@ Benchmark suite executing with the following configuration: -__Input: long_list__ - -Run Time - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameIPSAverageDevitationMedian99th %
flat_map2.59 K386.13 µs±81.99%249.25 µs1294.67 µs
map.flatten2.24 K446.12 µs±34.33%389.54 µs900.29 µs
- - -Run Time Comparison - - - - - - - - - - - - - - - - - - -
NameIPSSlower
flat_map2.59 K 
map.flatten2.24 K1.16x
- - - -Memory Usage - - - - - - - - - - - - - - - - - -
NameAverageFactor
flat_map625 KB 
map.flatten781.25 KB1.25x
- - - -__Input: short_list__ - Run Time @@ -145,20 +63,20 @@ Run Time - - - - - + + + + + - - - - - + + + + +
flat_map7.94 M126.01 ns±47.04%120.90 ns145.80 ns3.23 K309.46 µs±34.59%259.75 µs560.50 µs
map.flatten7.44 M134.46 ns±43.20%125 ns187.50 ns2.39 K418.13 µs±18.55%398.54 µs602.67 µs
@@ -173,14 +91,14 @@ Run Time Comparison Slower flat_map - 7.94 M + 3.23 K   map.flatten - 7.44 M - 1.07x + 2.39 K + 1.35x @@ -197,12 +115,12 @@ Memory Usage flat_map - 640 B + 781.22 KB   map.flatten - 800 B - 1.25x + 937.50 KB + 1.2x \ No newline at end of file diff --git a/examples/bench/no_input/README.md b/examples/bench/no_input/README.md new file mode 100644 index 0000000..92b56c6 --- /dev/null +++ b/examples/bench/no_input/README.md @@ -0,0 +1,208 @@ +Benchmark Example + +Benchmark run from 2023-12-17 06:53:53.740071Z UTC + +## System + +Benchmark suite executing on the following system: + + + + + + + + + + + + + + + + + + + + + +
Operating SystemmacOS
CPU InformationApple M1
Number of Available Cores8
Available Memory16 GB
Elixir Version1.15.7
Erlang Version26.2
+ +## Configuration + +Benchmark suite executing with the following configuration: + + + + + + + + + + + + +
:time10 ms
:parallel1
:warmup20 ms
+ +## Statistics + + + +__Input: long_list__ + +Run Time + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameIPSAverageDevitationMedian99th %
flat_map3.85 K259.90 µs±29.67%226.92 µs509 µs
map.flatten2.65 K377.94 µs±21.63%372.46 µs567.92 µs
+ + +Run Time Comparison + + + + + + + + + + + + + + + + + + +
NameIPSSlower
flat_map3.85 K 
map.flatten2.65 K1.45x
+ + + +Memory Usage + + + + + + + + + + + + + + + + + +
NameAverageFactor
flat_map625 KB 
map.flatten781.25 KB1.25x
+ + + +__Input: short_list__ + +Run Time + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameIPSAverageDevitationMedian99th %
flat_map7.90 M126.51 ns±45.72%120.90 ns146.35 ns
map.flatten7.47 M133.86 ns±42.20%125 ns187.50 ns
+ + +Run Time Comparison + + + + + + + + + + + + + + + + + + +
NameIPSSlower
flat_map7.90 M 
map.flatten7.47 M1.06x
+ + + +Memory Usage + + + + + + + + + + + + + + + + + +
NameAverageFactor
flat_map640 B 
map.flatten800 B1.25x
\ No newline at end of file diff --git a/examples/bench/no_input/run.exs b/examples/bench/no_input/run.exs new file mode 100644 index 0000000..c3a7c44 --- /dev/null +++ b/examples/bench/no_input/run.exs @@ -0,0 +1,21 @@ +map_fun = fn i -> [i, i * i] end + +Benchee.run( + %{ + "flat_map" => fn -> + 1..10_000 |> Enum.to_list() |> Enum.flat_map(map_fun) + end, + "map.flatten" => fn -> + 1..10_000 |> Enum.to_list() |> Enum.map(map_fun) |> List.flatten() + end + }, + title: "Benchmark Example", + time: 0.01, + memory_time: 0.01, + warmup: 0.02, + print: [fast_warning: false], + formatters: [ + {Benchee.Formatters.Markdown, file: "bench/basic/README.md"}, + Benchee.Formatters.Console + ] +) diff --git a/lib/benchee/formatters/markdown/helpers.ex b/lib/benchee/formatters/markdown/helpers.ex index 45e849b..a782d7b 100644 --- a/lib/benchee/formatters/markdown/helpers.ex +++ b/lib/benchee/formatters/markdown/helpers.ex @@ -41,10 +41,14 @@ defmodule Benchee.Formatter.Markdown.Helpers do Enum.filter(suite.scenarios, fn scenario -> scenario.input_name == input end) end + def inputs(%{configuration: %{input_names: names}}), do: names + def inputs(suite) do suite.configuration.inputs |> Enum.map(fn {name, _} -> name end) end + def inputs?(%{configuration: %{input_names: names}}), do: !Enum.empty?(names) |> IO.inspect() + def inputs?(suite), do: is_list(suite.configuration.inputs) def comparison?(suite), do: length(suite.scenarios) > 1 diff --git a/mix.exs b/mix.exs index fbf6220..0851c60 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule BencheeMarkdown.MixProject do use Mix.Project @source_url "https://github.com/hrzndhrn/benchee_markdown" - @version "0.3.2" + @version "0.3.3" def project do [ From 8a1620d6eb8fda3dcf39ba5c3548774ff85f064e Mon Sep 17 00:00:00 2001 From: Marcus Kruse Date: Sun, 17 Dec 2023 09:14:03 +0100 Subject: [PATCH 2/5] Make credo happy --- lib/benchee/formatters/markdown/helpers.ex | 2 +- mix.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/benchee/formatters/markdown/helpers.ex b/lib/benchee/formatters/markdown/helpers.ex index a782d7b..3e7dfd0 100644 --- a/lib/benchee/formatters/markdown/helpers.ex +++ b/lib/benchee/formatters/markdown/helpers.ex @@ -47,7 +47,7 @@ defmodule Benchee.Formatter.Markdown.Helpers do suite.configuration.inputs |> Enum.map(fn {name, _} -> name end) end - def inputs?(%{configuration: %{input_names: names}}), do: !Enum.empty?(names) |> IO.inspect() + def inputs?(%{configuration: %{input_names: names}}), do: !Enum.empty?(names) def inputs?(suite), do: is_list(suite.configuration.inputs) diff --git a/mix.lock b/mix.lock index 47944bf..e15af91 100644 --- a/mix.lock +++ b/mix.lock @@ -4,15 +4,15 @@ "credo": {:hex, :credo, "1.7.1", "6e26bbcc9e22eefbff7e43188e69924e78818e2fe6282487d0703652bc20fd62", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e9871c6095a4c0381c89b6aa98bc6260a8ba6addccf7f6a53da8849c748a58a2"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.4.2", "764a6e8e7a354f0ba95d58418178d486065ead1f69ad89782817c296d0d746a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "516603d8067b2fd585319e4b13d3674ad4f314a5902ba8130cd97dc902ce6bbd"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.38", "b42252eddf63bda05554ba8be93a1262dc0920c721f1aaf989f5de0f73a2e367", [:mix], [], "hexpm", "2cd0907795aaef0c7e8442e376633c5b3bd6edc8dbbdc539b22f095501c1cdb6"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "ex_doc": {:hex, :ex_doc, "0.30.9", "d691453495c47434c0f2052b08dd91cc32bc4e1a218f86884563448ee2502dd2", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "d7aaaf21e95dc5cddabf89063327e96867d00013963eadf2c6ad135506a8bc10"}, + "ex_doc": {:hex, :ex_doc, "0.31.0", "06eb1dfd787445d9cab9a45088405593dd3bb7fe99e097eaa71f37ba80c7a676", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5350cafa6b7f77bdd107aa2199fe277acf29d739aba5aee7e865fc680c62a110"}, "excoveralls": {:hex, :excoveralls, "0.18.0", "b92497e69465dc51bc37a6422226ee690ab437e4c06877e836f1c18daeb35da9", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1109bb911f3cb583401760be49c02cbbd16aed66ea9509fc5479335d284da60b"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, - "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, + "makeup_erlang": {:hex, :makeup_erlang, "0.1.3", "d684f4bac8690e70b06eb52dad65d26de2eefa44cd19d64a8095e1417df7c8fd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "b78dc853d2e670ff6390b605d807263bf606da3c82be37f9d7f68635bd886fc9"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, } From 9746b3a687a1ba20b23c575bb808ce391f6a4b16 Mon Sep 17 00:00:00 2001 From: Marcus Kruse Date: Sun, 24 Dec 2023 10:34:04 +0100 Subject: [PATCH 3/5] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d623ae9..dc7ed6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.3.3 - 2023-12-17 +## 0.3.3 - 2023-12-24 Changes for `benchee` 1.3. From 830f668fae3419392e2f057015f9a9d5acef1db6 Mon Sep 17 00:00:00 2001 From: Marcus Kruse Date: Sun, 24 Dec 2023 10:42:54 +0100 Subject: [PATCH 4/5] Add comment --- lib/benchee/formatters/markdown/helpers.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/benchee/formatters/markdown/helpers.ex b/lib/benchee/formatters/markdown/helpers.ex index 3e7dfd0..d3e11a6 100644 --- a/lib/benchee/formatters/markdown/helpers.ex +++ b/lib/benchee/formatters/markdown/helpers.ex @@ -47,8 +47,9 @@ defmodule Benchee.Formatter.Markdown.Helpers do suite.configuration.inputs |> Enum.map(fn {name, _} -> name end) end + # This definition uses `input_names` added in `benchee` 1.3.0. def inputs?(%{configuration: %{input_names: names}}), do: !Enum.empty?(names) - + # This definition can be removed when we use `benchee` version 1.3.0 as minimum. def inputs?(suite), do: is_list(suite.configuration.inputs) def comparison?(suite), do: length(suite.scenarios) > 1 From 8b8b882b1ba363b942fea733b8ee97cdbb4c4442 Mon Sep 17 00:00:00 2001 From: Marcus Kruse Date: Sun, 24 Dec 2023 11:02:20 +0100 Subject: [PATCH 5/5] Fix line length --- lib/benchee/formatters/markdown/helpers.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/benchee/formatters/markdown/helpers.ex b/lib/benchee/formatters/markdown/helpers.ex index d3e11a6..d2679be 100644 --- a/lib/benchee/formatters/markdown/helpers.ex +++ b/lib/benchee/formatters/markdown/helpers.ex @@ -49,7 +49,8 @@ defmodule Benchee.Formatter.Markdown.Helpers do # This definition uses `input_names` added in `benchee` 1.3.0. def inputs?(%{configuration: %{input_names: names}}), do: !Enum.empty?(names) - # This definition can be removed when we use `benchee` version 1.3.0 as minimum. + # This definition can be removed when we use `benchee` version 1.3.0 as + # minimum. def inputs?(suite), do: is_list(suite.configuration.inputs) def comparison?(suite), do: length(suite.scenarios) > 1