Skip to content

Commit

Permalink
Test difference among benchmark executions
Browse files Browse the repository at this point in the history 
According to the order of run, we have 14% difference among executions
for `(a:, **kwargs)`

### GitHub main tested before local

```
$ bundle exec ruby benchmarks.rb 

Will BENCHMARK_GEMS:
	memo_wise-github-main (1.10.0)
	memo_wise-local (1.10.0.dev)
ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-darwin23]
Warming up --------------------------------------
memo_wise-github-main (1.10.0): (a:, **kwargs)
                       125.193k i/100ms
memo_wise-local (1.10.0.dev): (a:, **kwargs)
                       143.649k i/100ms
Calculating -------------------------------------
memo_wise-github-main (1.10.0): (a:, **kwargs)
                          1.146M (±23.0%) i/s  (872.35 ns/i) -      5.508M in   5.067507s
memo_wise-local (1.10.0.dev): (a:, **kwargs)
                          1.309M (±18.0%) i/s  (764.21 ns/i) -      6.464M in   5.169169s

Comparison:
memo_wise-local (1.10.0.dev): (a:, **kwargs):  1308539.2 i/s
memo_wise-github-main (1.10.0): (a:, **kwargs):  1146322.9 i/s - same-ish: difference falls within error
```

|Method arguments|`memo_wise-github-main` (1.10.0)|
|--|--|
|`(a:, **kwargs)`|1.14x|

### local tested before GitHub main
```
$ bundle exec ruby benchmarks.rb 

Will BENCHMARK_GEMS:
	memo_wise-local (1.10.0.dev)
	memo_wise-github-main (1.10.0)
ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-darwin23]
Warming up --------------------------------------
memo_wise-local (1.10.0.dev): (a:, **kwargs)
                       130.824k i/100ms
memo_wise-github-main (1.10.0): (a:, **kwargs)
                       150.842k i/100ms
Calculating -------------------------------------
memo_wise-local (1.10.0.dev): (a:, **kwargs)
                          1.184M (±23.6%) i/s  (844.93 ns/i) -      5.756M in   5.139460s
memo_wise-github-main (1.10.0): (a:, **kwargs)
                          1.381M (±17.8%) i/s  (723.94 ns/i) -      6.637M in   5.026159s

Comparison:
memo_wise-github-main (1.10.0): (a:, **kwargs):  1381332.2 i/s
memo_wise-local (1.10.0.dev): (a:, **kwargs):  1183526.0 i/s - same-ish: difference falls within error
```

|Method arguments|`memo_wise-github-main` (1.10.0)|
|--|--|
|`(a:, **kwargs)`|0.86x|

Ref: panorama-ed#349
  • Loading branch information
@tagliala
tagliala committed Sep 22, 2024
1 parent 40cb797 commit b849956
Show file tree
Hide file tree
Showing 3 changed files with 3 additions and 94 deletions.
4 changes: 2 additions & 2 deletions benchmarks/Gemfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
ruby ">= 2.7.7"

gem "benchmark-ips", "2.14.0"
gem "gem_bench", "2.0.3"
gem "gem_bench", "2.0.5"

# NOTE: Regarding `require: false` below
# 1. GitHub version of MemoWise and the local source of MemoWise share a namespace
Expand All @@ -28,4 +28,4 @@ else
gem "memoizer", "1.0.3"
end

gem "memo_wise", github: "panorama-ed/memo_wise", branch: "main", require: false
gem "memo_wise", github: "tagliala/memo_wise", branch: "chore/prefer-require-relative", require: false
91 changes: 0 additions & 91 deletions benchmarks/benchmarks.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,36 +26,6 @@
memoization_method: :memo_wise,
},
),
GemBench::Jersey.new(
gem_name: "alt_memery",
trades: {
"Memery" => "AltMemery"
},
metadata: {
activation_code: "include AltMemery",
memoization_method: :memoize,
},
),
GemBench::Jersey.new(
gem_name: "memoist3",
trades: {
"Memoist" => "MemoistThree"
},
metadata: {
activation_code: "extend MemoistThree",
memoization_method: :memoize,
},
),
GemBench::Jersey.new(
gem_name: "memoist",
trades: {
"Memoist" => "MemoistOne"
},
metadata: {
activation_code: "extend MemoistOne",
memoization_method: :memoize,
},
),
].each(&:doff_and_don) # Copies, re-namespaces, and requires each gem.

# We've already installed the `memo_wise` version on the `main` branch from GitHub in the
Expand Down Expand Up @@ -119,11 +89,6 @@ def benchmark_name
end
benchmarked_gems.push(
BenchmarkGem.new(MemoWise, "prepend MemoWise", :memo_wise, LOCAL_BENCHMARK_NAME),
(BenchmarkGem.new(DDMemoize, "DDMemoize.activate(self)", :memoize, "ddmemoize") if defined?(DDMemoize)),
(BenchmarkGem.new(Dry::Core, "include Dry::Core::Memoizable", :memoize, "dry-core") if defined?(Dry::Core)),
(BenchmarkGem.new(Memery, "include Memery", :memoize, "memery") if defined?(Memery)),
(BenchmarkGem.new(Memoized, "include Memoized", :memoize, "memoized") if defined?(Memoized)),
(BenchmarkGem.new(Memoizer, "include Memoizer", :memoize, "memoizer") if defined?(Memoizer))
)
BENCHMARK_GEMS = benchmarked_gems.compact.shuffle

Expand Down Expand Up @@ -189,69 +154,13 @@ def positional_splat_keyword_and_double_splat_args(a, *args, b:, **kwargs)
# Each method within these benchmarks is initially run once to memoize the
# result value, so our benchmark only tests memoized retrieval time.
benchmark_lambdas = [
lambda do |x, instance, benchmark_gem|
instance.no_args

x.report("#{benchmark_gem.benchmark_name}: ()") do
instance.no_args
end
end,
lambda do |x, instance, benchmark_gem|
instance.one_positional_arg(1)

x.report("#{benchmark_gem.benchmark_name}: (a)") do
instance.one_positional_arg(1)
end
end,
lambda do |x, instance, benchmark_gem|
instance.positional_args(1, 2)

x.report("#{benchmark_gem.benchmark_name}: (a, b)") do
instance.positional_args(1, 2)
end
end,
lambda do |x, instance, benchmark_gem|
instance.one_keyword_arg(a: 1)

x.report("#{benchmark_gem.benchmark_name}: (a:)") do
instance.one_keyword_arg(a: 1)
end
end,
lambda do |x, instance, benchmark_gem|
instance.keyword_args(a: 1, b: 2)

x.report("#{benchmark_gem.benchmark_name}: (a:, b:)") do
instance.keyword_args(a: 1, b: 2)
end
end,
lambda do |x, instance, benchmark_gem|
instance.positional_and_keyword_args(1, b: 2)

x.report("#{benchmark_gem.benchmark_name}: (a, b:)") do
instance.positional_and_keyword_args(1, b: 2)
end
end,
lambda do |x, instance, benchmark_gem|
instance.positional_and_splat_args(1, 2)

x.report("#{benchmark_gem.benchmark_name}: (a, *args)") do
instance.positional_and_splat_args(1, 2)
end
end,
lambda do |x, instance, benchmark_gem|
instance.keyword_and_double_splat_args(a: 1, b: 2)

x.report("#{benchmark_gem.benchmark_name}: (a:, **kwargs)") do
instance.keyword_and_double_splat_args(a: 1, b: 2)
end
end,
lambda do |x, instance, benchmark_gem|
instance.positional_splat_keyword_and_double_splat_args(1, 2, b: 3, a: 4)

x.report("#{benchmark_gem.benchmark_name}: (a, *args, b:, **kwargs)") do
instance.positional_splat_keyword_and_double_splat_args(1, 2, b: 3, a: 4)
end
end
]

# We benchmark different cases separately, to ensure that slow performance in
Expand Down
2 changes: 1 addition & 1 deletion lib/memo_wise/version.rb
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# frozen_string_literal: true

module MemoWise
VERSION = "1.10.0"
VERSION = "1.10.0.dev"
end

0 comments on commit b849956

Please sign in to comment.