Test difference among benchmark executions

According to the order of run, we have 14% difference among executions for `(a:, **kwargs)` ### GitHub main tested before local ``` $ bundle exec ruby benchmarks.rb Will BENCHMARK_GEMS: memo_wise-github-main (1.10.0) memo_wise-local (1.10.0.dev) ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-darwin23] Warming up -------------------------------------- memo_wise-github-main (1.10.0): (a:, **kwargs) 125.193k i/100ms memo_wise-local (1.10.0.dev): (a:, **kwargs) 143.649k i/100ms Calculating ------------------------------------- memo_wise-github-main (1.10.0): (a:, **kwargs) 1.146M (±23.0%) i/s (872.35 ns/i) - 5.508M in 5.067507s memo_wise-local (1.10.0.dev): (a:, **kwargs) 1.309M (±18.0%) i/s (764.21 ns/i) - 6.464M in 5.169169s Comparison: memo_wise-local (1.10.0.dev): (a:, **kwargs): 1308539.2 i/s memo_wise-github-main (1.10.0): (a:, **kwargs): 1146322.9 i/s - same-ish: difference falls within error |Method arguments|`memo_wise-github-main` (1.10.0)| |--|--| |`(a:, **kwargs)`|1.14x| |Method arguments|| |--| |`(a:, **kwargs)`|| ``` ### local tested before GitHub main ``` $ bundle exec ruby benchmarks.rb Will BENCHMARK_GEMS: memo_wise-local (1.10.0.dev) memo_wise-github-main (1.10.0) ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-darwin23] Warming up -------------------------------------- memo_wise-local (1.10.0.dev): (a:, **kwargs) 130.824k i/100ms memo_wise-github-main (1.10.0): (a:, **kwargs) 150.842k i/100ms Calculating ------------------------------------- memo_wise-local (1.10.0.dev): (a:, **kwargs) 1.184M (±23.6%) i/s (844.93 ns/i) - 5.756M in 5.139460s memo_wise-github-main (1.10.0): (a:, **kwargs) 1.381M (±17.8%) i/s (723.94 ns/i) - 6.637M in 5.026159s Comparison: memo_wise-github-main (1.10.0): (a:, **kwargs): 1381332.2 i/s memo_wise-local (1.10.0.dev): (a:, **kwargs): 1183526.0 i/s - same-ish: difference falls within error |Method arguments|`memo_wise-github-main` (1.10.0)| |--|--| |`(a:, **kwargs)`|0.86x| |Method arguments|| |--| |`(a:, **kwargs)`|| ``` Ref: panorama-ed#349
tagliala · Sep 22, 2024 · 7eed3f8 · 7eed3f8
1 parent 40cb797
commit 7eed3f8
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 94 deletions.
diff --git a/benchmarks/Gemfile b/benchmarks/Gemfile
@@ -7,7 +7,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby ">= 2.7.7"
 
 gem "benchmark-ips", "2.14.0"
-gem "gem_bench", "2.0.3"
+gem "gem_bench", "2.0.5"
 
 # NOTE: Regarding `require: false` below
 # 1. GitHub version of MemoWise and the local source of MemoWise share a namespace
@@ -28,4 +28,4 @@ else
   gem "memoizer", "1.0.3"
 end
 
-gem "memo_wise", github: "panorama-ed/memo_wise", branch: "main", require: false
+gem "memo_wise", github: "tagliala/memo_wise", branch: "chore/prefer-require-relative", require: false
diff --git a/benchmarks/benchmarks.rb b/benchmarks/benchmarks.rb
@@ -26,36 +26,6 @@
       memoization_method: :memo_wise,
     },
   ),
-  GemBench::Jersey.new(
-    gem_name: "alt_memery",
-    trades: {
-      "Memery" => "AltMemery"
-    },
-    metadata: {
-      activation_code: "include AltMemery",
-      memoization_method: :memoize,
-    },
-  ),
-  GemBench::Jersey.new(
-    gem_name: "memoist3",
-    trades: {
-      "Memoist" => "MemoistThree"
-    },
-    metadata: {
-      activation_code: "extend MemoistThree",
-      memoization_method: :memoize,
-    },
-  ),
-  GemBench::Jersey.new(
-    gem_name: "memoist",
-    trades: {
-      "Memoist" => "MemoistOne"
-    },
-    metadata: {
-      activation_code: "extend MemoistOne",
-      memoization_method: :memoize,
-    },
-  ),
 ].each(&:doff_and_don) # Copies, re-namespaces, and requires each gem.
 
 # We've already installed the `memo_wise` version on the `main` branch from GitHub in the
@@ -119,11 +89,6 @@ def benchmark_name
 end
 benchmarked_gems.push(
   BenchmarkGem.new(MemoWise, "prepend MemoWise", :memo_wise, LOCAL_BENCHMARK_NAME),
-  (BenchmarkGem.new(DDMemoize, "DDMemoize.activate(self)", :memoize, "ddmemoize") if defined?(DDMemoize)),
-  (BenchmarkGem.new(Dry::Core, "include Dry::Core::Memoizable", :memoize, "dry-core") if defined?(Dry::Core)),
-  (BenchmarkGem.new(Memery, "include Memery", :memoize, "memery") if defined?(Memery)),
-  (BenchmarkGem.new(Memoized, "include Memoized", :memoize, "memoized") if defined?(Memoized)),
-  (BenchmarkGem.new(Memoizer, "include Memoizer", :memoize, "memoizer") if defined?(Memoizer))
 )
 BENCHMARK_GEMS = benchmarked_gems.compact.shuffle
 
@@ -189,69 +154,13 @@ def positional_splat_keyword_and_double_splat_args(a, *args, b:, **kwargs)
 # Each method within these benchmarks is initially run once to memoize the
 # result value, so our benchmark only tests memoized retrieval time.
 benchmark_lambdas = [
-  lambda do |x, instance, benchmark_gem|
-    instance.no_args
-
-    x.report("#{benchmark_gem.benchmark_name}: ()") do
-      instance.no_args
-    end
-  end,
-  lambda do |x, instance, benchmark_gem|
-    instance.one_positional_arg(1)
-
-    x.report("#{benchmark_gem.benchmark_name}: (a)") do
-      instance.one_positional_arg(1)
-    end
-  end,
-  lambda do |x, instance, benchmark_gem|
-    instance.positional_args(1, 2)
-
-    x.report("#{benchmark_gem.benchmark_name}: (a, b)") do
-      instance.positional_args(1, 2)
-    end
-  end,
-  lambda do |x, instance, benchmark_gem|
-    instance.one_keyword_arg(a: 1)
-
-    x.report("#{benchmark_gem.benchmark_name}: (a:)") do
-      instance.one_keyword_arg(a: 1)
-    end
-  end,
-  lambda do |x, instance, benchmark_gem|
-    instance.keyword_args(a: 1, b: 2)
-
-    x.report("#{benchmark_gem.benchmark_name}: (a:, b:)") do
-      instance.keyword_args(a: 1, b: 2)
-    end
-  end,
-  lambda do |x, instance, benchmark_gem|
-    instance.positional_and_keyword_args(1, b: 2)
-
-    x.report("#{benchmark_gem.benchmark_name}: (a, b:)") do
-      instance.positional_and_keyword_args(1, b: 2)
-    end
-  end,
-  lambda do |x, instance, benchmark_gem|
-    instance.positional_and_splat_args(1, 2)
-
-    x.report("#{benchmark_gem.benchmark_name}: (a, *args)") do
-      instance.positional_and_splat_args(1, 2)
-    end
-  end,
   lambda do |x, instance, benchmark_gem|
     instance.keyword_and_double_splat_args(a: 1, b: 2)
 
     x.report("#{benchmark_gem.benchmark_name}: (a:, **kwargs)") do
       instance.keyword_and_double_splat_args(a: 1, b: 2)
     end
   end,
-  lambda do |x, instance, benchmark_gem|
-    instance.positional_splat_keyword_and_double_splat_args(1, 2, b: 3, a: 4)
-
-    x.report("#{benchmark_gem.benchmark_name}: (a, *args, b:, **kwargs)") do
-      instance.positional_splat_keyword_and_double_splat_args(1, 2, b: 3, a: 4)
-    end
-  end
 ]
 
 # We benchmark different cases separately, to ensure that slow performance in

diff --git a/lib/memo_wise/version.rb b/lib/memo_wise/version.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MemoWise
-  VERSION = "1.10.0"
+  VERSION = "1.10.0.dev"
 end