Cleanup

Can now test various scenarios from bin/bench by controlling
environment variables passed to the tests.  (The tests still handle
the number of iterations for efficiency of shelling out.)

This lets us do something like

```ruby
scenarios = {
  'caching on: caching serializers' => 'CACHE_ON=true CACHING_SERIALIZER=true',
  'caching off: caching serializers' => 'CACHE_ON=false CACHING_SERIALIZER=true',
  'caching on: non-caching serializers' => 'CACHE_ON=true CACHING_SERIALIZER=false',
  'caching off: non-caching serializers' => 'CACHE_ON=false CACHING_SERIALIZER=false'
}
Benchmark do |x|
  scenarios.each do |name, env_vars|
    x.report(name) { benchmark_tests(env_vars) }
  end
end
```

(Also configurable via env: TIMES, DEBUG)

Author: bf4

bin/bench

@@ -3,79 +3,138 @@ require 'fileutils'
 require 'benchmark'
 require 'pathname'
 require 'shellwords'
+require 'English'
+
+############################
+# USAGE
+#
+# bin/bench <ref1> <ref2>
+#   <ref1> defaults to the current branch
+#   <ref2> defaults to the master branch
+#   bin/branch current will run on the current branch
+###########################
+
+class Benchmarking
+  ROOT = Pathname File.expand_path(['..', '..'].join(File::Separator), __FILE__)
+  TMP_DIR = File.join(ROOT, 'tmp/bench')
+
+  attr_reader :prepend, :append
+
+  def initialize(prepend: '', append: '')
+    @prepend = prepend
+    @append = append
+    refresh_temp_dir if temp_dir_empty?
+  end
 
-ROOT = Pathname File.expand_path(['..', '..'].join(File::Separator), __FILE__)
-TMP_DIR = File.join(ROOT, 'tmp/bench')
+  def temp_dir_empty?
+    Dir[File.join(TMP_DIR, '*')].none?
+  end
 
-def temp_dir_empty?
-  Dir[File.join(TMP_DIR, '*')].none?
-end
+  def empty_temp_dir
+    FileUtils.mkdir_p(TMP_DIR)
+    Dir[File.join(TMP_DIR, '*')].each do |file|
+      FileUtils.rm(file)
+    end
+  end
 
-def empty_temp_dir
-  FileUtils.mkdir_p(TMP_DIR)
-  Dir[File.join(TMP_DIR, '*')].each do |file|
-    FileUtils.rm(file)
+  def fill_temp_dir
+    Dir[File.join(ROOT, 'test', 'benchmark', '*.rb')].each do |file|
+      FileUtils.cp(file, File.join(TMP_DIR, File.basename(file)))
+    end
+    at_exit { empty_temp_dir }
   end
-end
 
-def fill_temp_dir
-  Dir[File.join(ROOT, 'test', 'benchmark', '*.rb')].each do |file|
-    FileUtils.cp(file, File.join(TMP_DIR, File.basename(file)))
+  def refresh_temp_dir
+    empty_temp_dir
+    fill_temp_dir
   end
-  at_exit { empty_temp_dir }
-end
 
-def refresh_temp_dir
-  empty_temp_dir
-  fill_temp_dir
-end
+  def benchmark_tests
+    tmp_dir = Shellwords.shellescape(TMP_DIR)
+    system("#{prepend} bundle exec ruby -Ilib:test #{tmp_dir}/*_benchmark.rb #{append}")
+  end
 
-def benchmark_tests
-  refresh_temp_dir if temp_dir_empty?
-  system("bundle exec ruby -Ilib:test #{Shellwords.shellescape(TMP_DIR)}/*_benchmark.rb")
-end
+  def current_branch
+    @current_branch ||= `cat .git/HEAD | cut -d/ -f3,4,5`.chomp
+  end
 
-def current_branch
-  @current_branch ||= `cat .git/HEAD | cut -d/ -f3,4,5`.chomp
-end
+  def checkout_ref(ref)
+    puts `git checkout #{ref}`.chomp
+    abort "Checkout failed: #{ref}, #{$CHILD_STATUS.exitstatus}" unless $CHILD_STATUS.success?
+  end
 
-def checkout_ref(ref)
-  puts `git checkout #{ref}`.chomp
-  abort "Checkout failed: #{ref}, #{$?.exitstatus}" unless $?.success?
-end
+  def benchmark_refs(ref1: nil, ref2: nil)
+    ref0 = current_branch
+    ref1 ||= current_branch
+    ref2 ||= 'master'
 
-def benchmark
-  refresh_temp_dir
-  ref  = current_branch
+    actual = run_benchmark_at_ref(ref1)
+    master = run_benchmark_at_ref(ref2)
 
-  actual = run_benchmark_at_ref ref
-  master = run_benchmark_at_ref 'master'
+    checkout_ref(ref0)
 
-  checkout_ref(ref)
+    <<-REPORT
 
-  puts "\n\nResults ============================\n"
-  puts "------------------------------------~> (Branch) MASTER"
-  puts master
-  puts "------------------------------------\n\n"
 
-  puts "------------------------------------~> (Actual Branch) #{ref}"
-  puts actual
-  puts "------------------------------------"
-end
+Results ============================
+------------------------------------~> (Branch) #{ref2.upcase}
+    #{master} (seconds)
+------------------------------------
 
-def run_benchmark
-  response = Benchmark.realtime {
-    benchmark_tests
-  }
-  benchmark_tests
-  response
-end
 
-def run_benchmark_at_ref(ref)
-  checkout_ref(ref)
-  run_benchmark
+
+------------------------------------~> (Actual Branch) #{ref1.upcase}
+    #{actual} (seconds)
+------------------------------------
+    REPORT
+  rescue Exception # rubocop:disable Lint/RescueException
+    checkout_ref(ref0)
+    raise
+  end
+
+  def run_benchmark
+    bundle
+    parse_measurement Benchmark.measure {
+      benchmark_tests
+    }
+  end
+
+  def run_benchmark_at_ref(ref)
+    checkout_ref(ref)
+    run_benchmark
+  end
+
+  def bundle
+    system('rm -f Gemfile.lock; bundle check || bundle --local --quiet || bundle --quiet')
+  end
+
+  def parse_measurement(measurement)
+    user = measurement.utime
+    system = measurement.stime
+    total = measurement.total
+    real = measurement.real
+    {
+      :real => real,
+      :total => total,
+      :user => user,
+      :system => system
+    }
+  end
 end
 
-if $0 == __FILE__
-  benchmark
+if $PROGRAM_NAME == __FILE__
+  # Example configuration using all options
+  #   benchmarking = Benchmarking.new(prepend: 'TIMES=1000 CACHE_ON=true CACHING_SERIALIZER=true DEBUG=true', append: '> /dev/null')
+  #   benchmarking = Benchmarking.new(prepend: 'CACHE_ON=false CACHING_SERIALIZER=false', append: '> /dev/null')
+  benchmarking = Benchmarking.new(append: '> /dev/null')
+  test_type = ARGV[0]
+  case test_type
+  when 'current'
+    puts "Ran in #{benchmarking.run_benchmark} seconds."
+  else
+    # Default: Compare current_branch to master
+    # Optionally: pass in two refs as args to `bin/bench`
+    # TODO: Consider checking across more revisions, to automatically find problems.
+    puts benchmarking.benchmark_refs(ref1: ARGV[0], ref2: ARGV[1])
+  end
 end

test/benchmark/benchmark_helper.rb

@@ -1,23 +1,32 @@
+# https://github.com/rails-api/active_model_serializers/pull/872
+# approx ref 792fb8a9053f8db3c562dae4f40907a582dd1720 to test against
 require 'bundler/setup'
 
 require 'rails'
-abort "Rails application already defined: #{Rails.application.class}" if Rails.application
+require 'active_model'
+require 'active_support'
+require 'active_support/json'
 require 'action_controller'
 require 'action_controller/test_case'
 require 'action_controller/railtie'
-require 'active_support/json'
+abort "Rails application already defined: #{Rails.application.class}" if Rails.application
 require 'minitest/autorun'
 # Ensure backward compatibility with Minitest 4
 Minitest::Test = MiniTest::Unit::TestCase unless defined?(Minitest::Test)
 
+# ref: https://gist.github.com/bf4/8744473
 class BenchmarkApp < Rails::Application
-  if Rails.version.to_s.start_with? '4'
-    config.action_controller.perform_caching = true
-    config.active_support.test_order         = :random
-    ActionController::Base.cache_store       = :memory_store
-    config.eager_load = false
-    config.secret_key_base = 'abc123'
-  end
+  config.action_controller.perform_caching = ENV['CACHE_ON'] != 'off'
+  ActionController::Base.cache_store       = :memory_store
+
+  # Set up production configuration
+  config.eager_load = true
+  config.cache_classes = true
+
+  config.active_support.test_order = :random
+  config.secret_token = '1234'
+  config.secret_key_base = 'abc123'
+  config.logger = Logger.new(IO::NULL)
 end
 
 require 'active_model_serializers'
@@ -38,5 +47,11 @@ def setup
   end
 end
 
+# Needs to initialize app before any serializes are defined, for sanity's sake.
+# Otherwise, you have to manually set perform caching.
+Rails.application.initialize!
+
 require_relative 'fixtures'
-BenchmarkApp.initialize!
+
+# Uncomment the below to test that cache is in use.
+# ActiveSupport::Cache::Store.logger = Logger.new(STDERR)

test/benchmark/fixtures.rb

@@ -12,9 +12,8 @@ class BlogSerializer < ActiveModel::Serializer
 class CommentSerializer < ActiveModel::Serializer
   attributes :id, :body
 
-  def custom_options
-    options
-  end
+  belongs_to :post
+  belongs_to :author
 end
 
 class PostSerializer < ActiveModel::Serializer
@@ -27,67 +26,81 @@ class PostSerializer < ActiveModel::Serializer
   def blog
     Blog.new(id: 999, name: 'Custom blog')
   end
-
-  def custom_options
-    options
-  end
 end
 
 class CachingAuthorSerializer < AuthorSerializer
-  cache key: 'writer'
+  cache key: 'writer', skip_digest: true
 end
 
 class CachingCommentSerializer < CommentSerializer
-  cache expires_in: 1.day
+  cache expires_in: 1.day, skip_digest: true
 end
 
 class CachingPostSerializer < PostSerializer
-  cache key: 'post', expires_in: 0.1
+  cache key: 'post', expires_in: 0.1, skip_digest: true
   belongs_to :blog, serializer: BlogSerializer
   belongs_to :author, serializer: CachingAuthorSerializer
   has_many :comments, serializer: CachingCommentSerializer
 end
 
-class Model
-  def initialize(hash = {})
-    @attributes = hash
+# ActiveModelSerializers::Model is a convenient
+# serializable class to inherit from when making
+# serializable non-activerecord objects.
+class BenchmarkModel
+  include ActiveModel::Model
+  include ActiveModel::Serializers::JSON
+
+  attr_reader :attributes
+
+  def initialize(attributes = {})
+    @attributes = attributes
+    super
   end
 
+  # Defaults to the downcased model name.
+  def id
+    attributes.fetch(:id) { self.class.name.downcase }
+  end
+
+  # Defaults to the downcased model name and updated_at
   def cache_key
-    "#{self.class.name.downcase}/#{id}-#{updated_at}"
+    attributes.fetch(:cache_key) { "#{self.class.name.downcase}/#{id}-#{updated_at.strftime("%Y%m%d%H%M%S%9N")}" }
   end
 
+  # Defaults to the time the serializer file was modified.
   def updated_at
-    @attributes[:updated_at] ||= Time.current.to_i
+    attributes.fetch(:updated_at) { File.mtime(__FILE__) }
   end
 
-  def read_attribute_for_serialization(name)
-    if name == :id || name == 'id'
-      id
+  def read_attribute_for_serialization(key)
+    if key == :id || key == 'id'
+      attributes.fetch(key) { id }
     else
-      @attributes[name]
+      attributes[key]
     end
   end
+end
 
-  def id
-    @attributes[:id] || @attributes['id'] || object_id
-  end
+class Comment < BenchmarkModel
+  attr_accessor :id, :body
 
-  def to_param
-    id
+  def cache_key
+    "#{self.class.name.downcase}/#{id}"
   end
+end
 
-  def method_missing(meth, *args)
-    if meth.to_s =~ /^(.*)=$/
-      @attributes[Regexp.last_match(1).to_sym] = args[0]
-    elsif @attributes.key?(meth)
-      @attributes[meth]
-    else
-      super
-    end
+class Author < BenchmarkModel
+  attr_accessor :id, :name, :posts
+end
+
+class Post < BenchmarkModel
+  attr_accessor :id, :title, :body, :comments, :blog, :author
+
+  def cache_key
+    'benchmarking::post/1-20151215212620000000000'
   end
 end
-class Comment < Model; end
-class Author < Model; end
-class Post < Model; end
-class Blog < Model; end
+
+class Blog < BenchmarkModel
+  attr_accessor :id, :name
+end