Setup benchmark testing

.rubocop.yml

@@ -1,6 +1,7 @@
 inherit_from: .rubocop_todo.yml
 
 AllCops:
+  TargetRubyVersion: 2.2
   Exclude:
     - config/initializers/forbidden_yaml.rb
     - !ruby/regexp /(vendor|bundle|bin|db|tmp)\/.*/

Gemfile

@@ -36,6 +36,11 @@ end
 # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
 gem 'tzinfo-data', platforms: (@windows_platforms + [:jruby])
 
+group :bench do
+  # https://github.com/rails-api/active_model_serializers/commit/cb4459580a6f4f37f629bf3185a5224c8624ca76
+  gem 'benchmark-ips', require: false, group: :development
+end
+
 group :test do
   gem 'sqlite3',                          platform: (@windows_platforms + [:ruby])
   gem 'activerecord-jdbcsqlite3-adapter', platform: :jruby

active_model_serializers.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0")
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
+  spec.executables   = []
 
   spec.required_ruby_version = '>= 2.0.0'
 

bin/bench

@@ -0,0 +1,170 @@
+#!/usr/bin/env ruby
+# ActiveModelSerializers Benchmark driver
+# Adapted from
+# https://github.com/ruby-bench/ruby-bench-suite/blob/8ad567f7e43a044ae48c36833218423bb1e2bd9d/rails/benchmarks/driver.rb
+require 'bundler'
+Bundler.setup
+require 'json'
+require 'pathname'
+require 'optparse'
+require 'digest'
+require 'pathname'
+require 'shellwords'
+require 'logger'
+require 'English'
+
+class BenchmarkDriver
+  ROOT = Pathname File.expand_path(File.join('..', '..'), __FILE__)
+  BASE = ENV.fetch('BASE') { ROOT.join('test', 'dummy') }
+  ESCAPED_BASE = Shellwords.shellescape(BASE)
+
+  def self.benchmark(options)
+    new(options).run
+  end
+
+  def self.parse_argv_and_run(argv = ARGV, options = {})
+    options = {
+      repeat_count: 1,
+      pattern: [],
+      env: 'CACHE_ON=on'
+    }.merge!(options)
+
+    OptionParser.new do |opts|
+      opts.banner = 'Usage: bin/bench [options]'
+
+      opts.on('-r', '--repeat-count [NUM]', 'Run benchmarks [NUM] times taking the best result') do |value|
+        options[:repeat_count] = value.to_i
+      end
+
+      opts.on('-p', '--pattern <PATTERN1,PATTERN2,PATTERN3>', 'Benchmark name pattern') do |value|
+        options[:pattern] = value.split(',')
+      end
+
+      opts.on('-e', '--env <var1=val1,var2=val2,var3=vale>', 'ENV variables to pass in') do |value|
+        options[:env] = value.split(',')
+      end
+    end.parse!(argv)
+
+    benchmark(options)
+  end
+
+  attr_reader :commit_hash, :base
+
+  # Based on logfmt:
+  # https://www.brandur.org/logfmt
+  # For more complete implementation see:
+  # see https://github.com/arachnid-cb/logfmtr/blob/master/lib/logfmtr/base.rb
+  # For usage see:
+  # https://blog.codeship.com/logfmt-a-log-format-thats-easy-to-read-and-write/
+  # https://engineering.heroku.com/blogs/2014-09-05-hutils-explore-your-structured-data-logs/
+  # For Ruby parser see:
+  # https://github.com/cyberdelia/logfmt-ruby
+  def self.summary_logger(device = 'output.txt')
+    require 'time'
+    logger = Logger.new(device)
+    logger.level = Logger::INFO
+    logger.formatter = proc { |severity, datetime, progname, msg|
+      msg = "'#{msg}'"
+      "level=#{severity} time=#{datetime.utc.iso8601(6)} pid=#{Process.pid} progname=#{progname} msg=#{msg}#{$INPUT_RECORD_SEPARATOR}"
+    }
+    logger
+  end
+
+  def self.stdout_logger
+    logger = Logger.new(STDOUT)
+    logger.level = Logger::INFO
+    logger.formatter = proc { |_, _, _, msg| "#{msg}#{$INPUT_RECORD_SEPARATOR}" }
+    logger
+  end
+
+  def initialize(options)
+    @writer       = ENV['SUMMARIZE'] ? self.class.summary_logger : self.class.stdout_logger
+    @repeat_count = options[:repeat_count]
+    @pattern      = options[:pattern]
+    @commit_hash  = options.fetch(:commit_hash) { `git rev-parse --short HEAD`.chomp }
+    @base         = options.fetch(:base) { ESCAPED_BASE }
+    @env          = Array(options[:env]).join(' ')
+    @rubyopt = options[:rubyopt] # TODO: rename
+  end
+
+  def run
+    files.each do |path|
+      next if !@pattern.empty? && /#{@pattern.join('|')}/ !~ File.basename(path)
+      run_single(Shellwords.shellescape(path))
+    end
+  end
+
+  private
+
+  def files
+    Dir[File.join(base, 'bm_*')]
+  end
+
+  def run_single(path)
+    script = "RAILS_ENV=production #{@env} ruby #{@rubyopt} #{path}"
+    environment = `ruby -v`.chomp.strip[/\d+\.\d+\.\d+\w+/]
+
+    runs_output = measure(script)
+    if runs_output.empty?
+      results = { error: :no_results }
+      return
+    end
+
+    results = {}
+    results['commit_hash'] = commit_hash
+    results['version'] = runs_output.first['version']
+    results['benchmark_run[environment]'] = environment
+    results['runs'] = []
+
+    runs_output.each do |output|
+      results['runs'] << {
+        'benchmark_type[category]' => output['label'],
+        'benchmark_run[result][iterations_per_second]' => output['iterations_per_second'].round(3),
+        'benchmark_run[result][total_allocated_objects_per_iteration]' => output['total_allocated_objects_per_iteration']
+      }
+    end
+  ensure
+    results && report(results)
+  end
+
+  def report(results)
+    @writer.info { 'Benchmark results:' }
+    @writer.info { JSON.pretty_generate(results) }
+  end
+
+  def summarize(result)
+    puts "#{result['label']} #{result['iterations_per_second']}/ips; #{result['total_allocated_objects_per_iteration']} objects"
+  end
+
+  # FIXME: ` provides the full output but it'll return failed output as well.
+  def measure(script)
+    results = Hash.new { |h, k| h[k] = [] }
+
+    @repeat_count.times do
+      output = sh(script)
+      output.each_line do |line|
+        next if line.nil?
+        begin
+          result = JSON.parse(line)
+        rescue JSON::ParserError
+          result = { error: line } # rubocop:disable Lint/UselessAssignment
+        else
+          summarize(result)
+          results[result['label']] << result
+        end
+      end
+    end
+
+    results.map do |_, bm_runs|
+      bm_runs.sort_by do |run|
+        run['iterations_per_second']
+      end.last
+    end
+  end
+
+  def sh(cmd)
+    `#{cmd}`
+  end
+end
+
+BenchmarkDriver.parse_argv_and_run if $PROGRAM_NAME == __FILE__

bin/serve_dummy

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -e
+
+case "$1" in
+
+  start)
+  config="${CONFIG_RU:-test/dummy/config.ru}"
+  bundle exec ruby -Ilib -S rackup "$config" --daemonize --pid tmp/dummy_app.pid --warn --server webrick
+  until [ -f 'tmp/dummy_app.pid' ]; do
+    sleep 0.1 # give it time to start.. I don't know a better way
+  done
+  cat tmp/dummy_app.pid
+  true
+  ;;
+
+  stop)
+  if [ -f 'tmp/dummy_app.pid' ]; then
+    kill -TERM $(cat tmp/dummy_app.pid)
+  else
+    echo 'No pidfile'
+    false
+  fi
+  ;;
+
+  status)
+  if [ -f 'tmp/dummy_app.pid' ]; then
+    kill -0 $(cat tmp/dummy_app.pid)
+    [ "$?" -eq 0 ]
+  else
+    echo 'No pidfile'
+    false
+  fi
+  ;;
+
+  *)
+  echo "Usage: $0 [start|stop|status]"
+  ;;
+
+esac

test/dummy/app.rb

@@ -0,0 +1,65 @@
+# https://github.com/rails-api/active_model_serializers/pull/872
+# approx ref 792fb8a9053f8db3c562dae4f40907a582dd1720 to test against
+require 'bundler/setup'
+
+require 'rails'
+require 'active_model'
+require 'active_support'
+require 'active_support/json'
+require 'action_controller'
+require 'action_controller/test_case'
+require 'action_controller/railtie'
+abort "Rails application already defined: #{Rails.application.class}" if Rails.application
+
+class NullLogger < Logger
+  def initialize(*_args)
+  end
+
+  def add(*_args, &_block)
+  end
+end
+class DummyLogger < ActiveSupport::Logger
+  def initialize
+    @file = StringIO.new
+    super(@file)
+  end
+
+  def messages
+    @file.rewind
+    @file.read
+  end
+end
+# ref: https://gist.github.com/bf4/8744473
+class DummyApp < Rails::Application
+  # Set up production configuration
+  config.eager_load = true
+  config.cache_classes = true
+  # CONFIG: CACHE_ON={on,off}
+  config.action_controller.perform_caching = ENV['CACHE_ON'] != 'off'
+  config.action_controller.cache_store = ActiveSupport::Cache.lookup_store(:memory_store)
+
+  config.active_support.test_order = :random
+  config.secret_token = 'S' * 30
+  config.secret_key_base = 'abc123'
+  config.consider_all_requests_local = false
+
+  # otherwise deadlock occured
+  config.middleware.delete 'Rack::Lock'
+
+  # to disable log files
+  config.logger = NullLogger.new
+  config.active_support.deprecation = :log
+  config.log_level = :info
+end
+
+require 'active_model_serializers'
+
+# Initialize app before any serializers are defined, for running across revisions.
+# ref: https://github.com/rails-api/active_model_serializers/pull/1478
+Rails.application.initialize!
+# HACK: Serializer::cache depends on the ActionController-dependent configs being set.
+ActiveSupport.on_load(:action_controller) do
+  require_relative 'fixtures'
+end
+
+require_relative 'controllers'

test/dummy/benchmarking_support.rb

@@ -0,0 +1,66 @@
+require 'benchmark/ips'
+require 'json'
+
+# Add benchmarking runner from ruby-bench-suite
+# https://github.com/ruby-bench/ruby-bench-suite/blob/master/rails/benchmarks/support/benchmark_rails.rb
+module Benchmark
+  module ActiveModelSerializers
+    module TestMethods
+      def request(method, path)
+        response = Rack::MockRequest.new(DummyApp).send(method, path)
+        if response.status.in?([404, 500])
+          fail "omg, #{method}, #{path}, '#{response.status}', '#{response.body}'"
+        end
+        response
+      end
+    end
+
+    # extend Benchmark with an `ams` method
+    def ams(label = nil, time:, disable_gc: true, warmup: 3, &block)
+      fail ArgumentError.new, 'block should be passed' unless block_given?
+
+      if disable_gc
+        GC.disable
+      else
+        GC.enable
+      end
+
+      report = Benchmark.ips(time, warmup, true) do |x|
+        x.report(label) { yield }
+      end
+
+      entry = report.entries.first
+
+      output = {
+        label: label,
+        version: ::ActiveModel::Serializer::VERSION.to_s,
+        iterations_per_second: entry.ips,
+        iterations_per_second_standard_deviation: entry.stddev_percentage,
+        total_allocated_objects_per_iteration: count_total_allocated_objects(&block)
+      }.to_json
+
+      puts output
+      output
+    end
+
+    def count_total_allocated_objects
+      if block_given?
+        key =
+          if RUBY_VERSION < '2.2'
+            :total_allocated_object
+          else
+            :total_allocated_objects
+          end
+
+        before = GC.stat[key]
+        yield
+        after = GC.stat[key]
+        after - before
+      else
+        -1
+      end
+    end
+  end
+
+  extend Benchmark::ActiveModelSerializers
+end