Setup benchmarking structure; benchmark caching

- Setup dummy app
- `bin/bench` etc adapted from ruby-bench-suite
- benchmark cache/no cache
- compare branches
- remove rake dependency that loads unnecessary files
  - run with
    - ./bin/bench
- remove git gem dependency

Gemfile

@@ -36,6 +36,11 @@ end
 # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
 gem 'tzinfo-data', platforms: (@windows_platforms + [:jruby])
 
+group :bench do
+  # https://github.com/rails-api/active_model_serializers/commit/cb4459580a6f4f37f629bf3185a5224c8624ca76
+  gem 'benchmark-ips', require: false, group: :development
+end
+
 group :test do
   gem 'sqlite3',                          platform: (@windows_platforms + [:ruby])
   gem 'activerecord-jdbcsqlite3-adapter', platform: :jruby
@@ -45,5 +50,4 @@ end
 
 group :development, :test do
   gem 'rubocop', '~> 0.34.0', require: false
-  gem 'git'
 end

Rakefile

@@ -73,38 +73,3 @@ end
 
 desc 'CI test task'
 task :ci => [:default]
-
-require 'git'
-require 'benchmark'
-Rake::TestTask.new :benchmark_tests do |t|
-  t.libs << "test"
-  t.test_files = FileList['test/**/*_benchmark.rb']
-  t.ruby_opts = ['-r./test/test_helper.rb']
-  t.verbose = true
-end
-
-task :benchmark do
-  @git = Git.init('.')
-  ref  = @git.current_branch
-
-  actual = run_benchmark_spec ref
-  master = run_benchmark_spec 'master'
-
-  @git.checkout(ref)
-
-  puts "\n\nResults ============================\n"
-  puts "------------------------------------~> (Branch) MASTER"
-  puts master
-  puts "------------------------------------\n\n"
-
-  puts "------------------------------------~> (Actual Branch) #{ref}"
-  puts actual
-  puts "------------------------------------"
-end
-
-def run_benchmark_spec(ref)
-  @git.checkout(ref)
-  response = Benchmark.realtime { Rake::Task['benchmark_tests'].invoke }
-  Rake::Task['benchmark_tests'].reenable
-  response
-end

active_model_serializers.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0")
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
+  spec.executables   = []
 
   spec.required_ruby_version = '>= 2.0.0'
 

bin/bench

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+# ActiveModelSerializers Benchmark driver
+# Adapted from
+# https://github.com/ruby-bench/ruby-bench-suite/blob/8ad567f7e43a044ae48c36833218423bb1e2bd9d/rails/benchmarks/driver.rb
+require 'bundler'
+Bundler.setup
+require 'json'
+require 'pathname'
+require 'optparse'
+require 'digest'
+require 'pathname'
+
+class BenchmarkDriver
+  ROOT = Pathname File.expand_path(File.join('..', '..'), __FILE__)
+  BASE = ENV.fetch('BASE') { ROOT.join('test', 'dummy') }
+
+  def self.benchmark(options)
+    self.new(options).run
+  end
+
+  def initialize(options)
+    @repeat_count = options[:repeat_count]
+    @pattern      = options[:pattern]
+    @env          = Array(options[:env]).join(' ')
+  end
+
+  def run
+    files.each do |path|
+      next if !@pattern.empty? && /#{@pattern.join('|')}/ !~ File.basename(path)
+      run_single(path)
+    end
+  end
+
+  private
+
+  def files
+    Dir[File.join(BASE, 'bm_*')]
+  end
+
+  def run_single(path)
+    script = "RAILS_ENV=production #{@env} ruby #{path}"
+    environment = `ruby -v`.chomp.strip[/\d+\.\d+\.\d+\w+/]
+    commit_hash = ENV['COMMIT_HASH'] || `git rev-parse --short HEAD`.chomp
+
+    runs_output = measure(script)
+
+    results = {}
+    results['commit_hash'] = commit_hash
+    results['version'] = runs_output.first['version']
+    results['benchmark_run[environment]'] = environment
+    results['runs'] = []
+
+    runs_output.each do |output|
+      results['runs'] << {
+        'benchmark_type[category]' => output["label"],
+        'benchmark_run[result][iterations_per_second]' => output['iterations_per_second'].round(3),
+        'benchmark_run[result][total_allocated_objects_per_iteration]' => output["total_allocated_objects_per_iteration"]
+      }
+    end
+  ensure
+    report(results)
+  end
+
+  def report(results)
+    puts 'Benchmark results:'
+    puts JSON.pretty_generate(results)
+  end
+
+  def summarize(result)
+    puts "#{result['label']} #{result['iterations_per_second']}/ips"
+  end
+
+  # FIXME: ` provides the full output but it'll return failed output as well.
+  def measure(script)
+    results = Hash.new {|h,k| h[k] = []}
+
+    @repeat_count.times do
+      output = `#{script}`
+      output.each_line do |line|
+        result = JSON.parse(line)
+        summarize(result)
+        results[result['label']] << result
+      end
+    end
+
+    results.map do |_, bm_runs|
+      bm_runs.sort_by do |run|
+        run['iterations_per_second']
+      end.last
+    end
+  end
+end
+
+options = {
+  repeat_count: 1,
+  pattern: [],
+  env: "CACHE_ON=on"
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: bin/bench [options]"
+
+  opts.on("-r", "--repeat-count [NUM]", "Run benchmarks [NUM] times taking the best result") do |value|
+    options[:repeat_count] = value.to_i
+  end
+
+  opts.on("-p", "--pattern <PATTERN1,PATTERN2,PATTERN3>", "Benchmark name pattern") do |value|
+    options[:pattern] = value.split(',')
+  end
+
+  opts.on("-e", "--env <var1=val1,var2=val2,var3=vale>", "ENV variables to pass in") do |value|
+    options[:env] = value.split(',')
+  end
+end.parse!(ARGV)
+
+BenchmarkDriver.benchmark(options)

test/dummy/app.rb

@@ -0,0 +1,94 @@
+# https://github.com/rails-api/active_model_serializers/pull/872
+# approx ref 792fb8a9053f8db3c562dae4f40907a582dd1720 to test against
+require 'bundler/setup'
+
+require 'rails'
+require 'active_model'
+require 'active_support'
+require 'active_support/json'
+require 'action_controller'
+require 'action_controller/test_case'
+require 'action_controller/railtie'
+abort "Rails application already defined: #{Rails.application.class}" if Rails.application
+
+class NullLogger < Logger
+  def initialize(*_args)
+  end
+
+  def add(*_args, &_block)
+  end
+end
+class DummyLogger < ActiveSupport::Logger
+  def initialize
+    @file = StringIO.new
+    super(@file)
+  end
+
+  def messages
+    @file.rewind
+    @file.read
+  end
+end
+# ref: https://gist.github.com/bf4/8744473
+class DummyApp < Rails::Application
+  # Set up production configuration
+  config.eager_load = true
+  config.cache_classes = true
+  # CONFIG: CACHE_ON={on,off}
+  config.action_controller.perform_caching = ENV['CACHE_ON'] != 'off'
+  config.action_controller.cache_store = ActiveSupport::Cache.lookup_store(:memory_store)
+
+  config.active_support.test_order = :random
+  config.secret_token = 'S' * 30
+  config.secret_key_base = 'abc123'
+  config.consider_all_requests_local = false
+
+  # otherwise deadlock occured
+  config.middleware.delete 'Rack::Lock'
+
+  # to disable log files
+  config.logger = NullLogger.new
+  config.active_support.deprecation = :log
+  config.log_level = :info
+end
+
+require 'active_model_serializers'
+
+# Initialize app before any serializers are defined, for sanity's sake.
+# Otherwise, you have to manually set perform caching.
+#
+# Details:
+#
+# 1. Upon load, when AMS.config.perform_caching is true,
+#    serializers inherit the cache store from ActiveModelSerializers.config.cache_store
+# 1. If the serializers are loaded before Rails is initialized (`Rails.application.initialize!`),
+#    these values are nil, and are not applied to the already loaded serializers
+# 1. If Rails is initialized before any serializers are loaded, then the configs are set,
+#    and are used when serializers are loaded
+# 1. In either case, `ActiveModelSerializers.config.cache_store`, and
+#    `ActiveModelSerializers.config.perform_caching` can be set at any time before the serializers
+#    are loaded,
+#    e.g.  `ActiveModel::Serializer.config.cache_store ||=
+#      ActiveSupport::Cache.lookup_store(ActionController::Base.cache_store ||
+#      Rails.cache || :memory_store)`
+#    and `ActiveModelSerializers.config.perform_caching = true`
+# 1. If the serializers are loaded before Rails is initialized, then,
+#    you can set the `_cache` store directly on the serializers.
+#    `ActiveModel::Serializer._cache ||=
+#      ActiveSupport::Cache.lookup_store(ActionController::Base.cache_store ||
+#      Rails.cache || :memory_store`
+#    is sufficient.
+#    Setting `_cache` to a truthy value will cause the CachedSerializer
+#    to consider it cached, which will apply to all serializers (bug? :bug: )
+#
+# This happens, in part, because the cache store is set for a serializer
+# when `cache` is called, and cache is usually called when the serializer is defined.
+#
+# So, there's now a 'workaround', something to debug, and a starting point.
+Rails.application.initialize!
+
+# HACK: Serializer::cache depends on the ActionController-dependent configs being set.
+ActiveSupport.on_load(:action_controller) do
+  require_relative 'fixtures'
+end
+require_relative 'controllers'

test/dummy/benchmarking_support.rb

@@ -0,0 +1,66 @@
+require 'benchmark/ips'
+require 'json'
+
+# Add benchmarking runner from ruby-bench-suite
+# https://github.com/ruby-bench/ruby-bench-suite/blob/master/rails/benchmarks/support/benchmark_rails.rb
+module Benchmark
+  module ActiveModelSerializers
+    module TestMethods
+      def request(method, path)
+        response = Rack::MockRequest.new(DummyApp).send(method, path)
+        if response.status.in?([404, 500])
+          fail "omg, #{method}, #{path}, '#{response.status}', '#{response.body}'"
+        end
+        response
+      end
+    end
+
+    # extend Benchmark with an `ams` method
+    def ams(label = nil, time:, disable_gc: true, warmup: 3, &block)
+      fail ArgumentError.new, 'block should be passed' unless block_given?
+
+      if disable_gc
+        GC.disable
+      else
+        GC.enable
+      end
+
+      report = Benchmark.ips(time, warmup, true) do |x|
+        x.report(label) { yield }
+      end
+
+      entry = report.entries.first
+
+      output = {
+        label: label,
+        version: ::ActiveModel::Serializer::VERSION.to_s,
+        iterations_per_second: entry.ips,
+        iterations_per_second_standard_deviation: entry.stddev_percentage,
+        total_allocated_objects_per_iteration: count_total_allocated_objects(&block)
+      }.to_json
+
+      puts output
+      output
+    end
+
+    def count_total_allocated_objects
+      if block_given?
+        key =
+          if RUBY_VERSION < '2.2'
+            :total_allocated_object
+          else
+            :total_allocated_objects
+          end
+
+        before = GC.stat[key]
+        yield
+        after = GC.stat[key]
+        after - before
+      else
+        -1
+      end
+    end
+  end
+
+  extend Benchmark::ActiveModelSerializers
+end