Merge pull request #326 from Shopify/executable

Add a bootsnap command to allow to precompile gems without booting the application

README.md

@@ -294,6 +294,19 @@ open    /c/nope.bundle -> -1
 # (nothing!)
 ```
 
+## Precompilation
+
+In development environments the bootsnap compilation cache is generated on the fly when source files are loaded.
+But in production environments, such as docker images, you might need to precompile the cache.
+
+To do so you can use the `bootsnap precompile` command.
+
+Example:
+
+```bash
+$ bundle exec bootsnap precompile --gemfile app/ lib/
+```
+
 ## When not to use Bootsnap
 
 *Alternative engines*: Bootsnap is pretty reliant on MRI features, and parts are disabled entirely on alternative ruby

bootsnap.gemspec

@@ -26,6 +26,9 @@ Gem::Specification.new do |spec|
   spec.files = %x(git ls-files -z ext lib).split("\x0") + %w(CHANGELOG.md LICENSE.txt README.md)
   spec.require_paths = %w(lib)
 
+  spec.bindir        = 'exe'
+  spec.executables   = %w(bootsnap)
+
   spec.required_ruby_version = '>= 2.3.0'
 
   if RUBY_PLATFORM =~ /java/

exe/bootsnap

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bootsnap/cli'
+exit Bootsnap::CLI.new(ARGV).run

lib/bootsnap/cli.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require 'bootsnap'
+require 'optparse'
+require 'fileutils'
+
+module Bootsnap
+  class CLI
+    unless Regexp.method_defined?(:match?)
+      module RegexpMatchBackport
+        refine Regepx do
+          def match?(string)
+            !!match(string)
+          end
+        end
+      end
+      using RegexpMatchBackport
+    end
+
+    attr_reader :cache_dir, :argv
+
+    attr_accessor :compile_gemfile, :exclude
+
+    def initialize(argv)
+      @argv = argv
+      self.cache_dir = 'tmp/cache'
+      self.compile_gemfile = false
+      self.exclude = nil
+    end
+
+    def precompile_command(*sources)
+      require 'bootsnap/compile_cache/iseq'
+
+      Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
+
+      if compile_gemfile
+        sources += $LOAD_PATH
+      end
+
+      sources.map { |d| File.expand_path(d) }.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_ruby_files(path).each do |ruby_file|
+            if !exclude || !exclude.match?(ruby_file)
+              CompileCache::ISeq.fetch(ruby_file, cache_dir: cache_dir)
+            end
+          end
+        end
+      end
+      0
+    end
+
+    dir_sort = begin
+      Dir['.', sort: false]
+      true
+    rescue ArgumentError
+      false
+    end
+
+    if dir_sort
+      def list_ruby_files(path)
+        if File.directory?(path)
+          Dir[File.join(path, '**/*.rb'), sort: false]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    else
+      def list_ruby_files(path)
+        if File.directory?(path)
+          Dir[File.join(path, '**/*.rb')]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    end
+
+    def run
+      parser.parse!(argv)
+      command = argv.shift
+      method = "#{command}_command"
+      if respond_to?(method)
+        public_send(method, *argv)
+      else
+        invalid_usage!("Unknown command: #{command}")
+      end
+    end
+
+    private
+
+    def invalid_usage!(message)
+      STDERR.puts message
+      STDERR.puts
+      STDERR.puts parser
+      1
+    end
+
+    def cache_dir=(dir)
+      @cache_dir = File.expand_path(File.join(dir, 'bootsnap-compile-cache'))
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: bootsnap COMMAND [ARGS]"
+        opts.separator ""
+        opts.separator "GLOBAL OPTIONS"
+        opts.separator ""
+
+        help = <<~EOS
+          Path to the bootsnap cache directory. Defaults to tmp/cache
+        EOS
+        opts.on('--cache-dir DIR', help.strip) do |dir|
+          self.cache_dir = dir
+        end
+
+        opts.separator ""
+        opts.separator "COMMANDS"
+        opts.separator ""
+        opts.separator "    precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
+
+        help = <<~EOS
+          Precompile the gems in Gemfile
+        EOS
+        opts.on('--gemfile', help) { self.compile_gemfile = true }
+
+        help = <<~EOS
+          Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
+        EOS
+        opts.on('--exclude PATTERN', help) { |pattern| self.exclude = Regexp.new(pattern) }
+      end
+    end
+  end
+end

lib/bootsnap/compile_cache/iseq.rb

@@ -26,6 +26,15 @@ def self.storage_to_output(binary, _args)
         end
       end
 
+      def self.fetch(path, cache_dir: ISeq.cache_dir)
+        Bootsnap::CompileCache::Native.fetch(
+          cache_dir,
+          path.to_s,
+          Bootsnap::CompileCache::ISeq,
+          nil,
+        )
+      end
+
       def self.input_to_output(_, _)
         nil # ruby handles this
       end
@@ -35,12 +44,7 @@ def load_iseq(path)
           # Having coverage enabled prevents iseq dumping/loading.
           return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
 
-          Bootsnap::CompileCache::Native.fetch(
-            Bootsnap::CompileCache::ISeq.cache_dir,
-            path.to_s,
-            Bootsnap::CompileCache::ISeq,
-            nil,
-          )
+          Bootsnap::CompileCache::ISeq.fetch(path.to_s)
         rescue Errno::EACCES
           Bootsnap::CompileCache.permission_error(path)
         rescue RuntimeError => e
@@ -61,6 +65,7 @@ def self.compile_option_updated
         crc = Zlib.crc32(option.inspect)
         Bootsnap::CompileCache::Native.compile_option_crc32 = crc
       end
+      compile_option_updated
 
       def self.install!(cache_dir)
         Bootsnap::CompileCache::ISeq.cache_dir = cache_dir

test/cli_test.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+require('test_helper')
+require('bootsnap/cli')
+
+module Bootsnap
+  class CLITest < Minitest::Test
+    include(TmpdirHelper)
+
+    def setup
+      super
+      @cache_dir = File.expand_path('tmp/cache/bootsnap-compile-cache')
+    end
+
+    def test_precompile_single_file
+      path = Help.set_file('a.rb', 'a = a = 3', 100)
+      CompileCache::ISeq.expects(:fetch).with(File.expand_path(path), cache_dir: @cache_dir)
+      assert_equal 0, CLI.new(['precompile', path]).run
+    end
+
+    def test_precompile_directory
+      path_a = Help.set_file('foo/a.rb', 'a = a = 3', 100)
+      path_b = Help.set_file('foo/b.rb', 'b = b = 3', 100)
+
+      CompileCache::ISeq.expects(:fetch).with(File.expand_path(path_a), cache_dir: @cache_dir)
+      CompileCache::ISeq.expects(:fetch).with(File.expand_path(path_b), cache_dir: @cache_dir)
+      assert_equal 0, CLI.new(['precompile', 'foo']).run
+    end
+
+    def test_precompile_exclude
+      path_a = Help.set_file('foo/a.rb', 'a = a = 3', 100)
+      path_b = Help.set_file('foo/b.rb', 'b = b = 3', 100)
+
+      CompileCache::ISeq.expects(:fetch).with(File.expand_path(path_a), cache_dir: @cache_dir)
+      assert_equal 0, CLI.new(['precompile', '--exclude', 'b.rb', 'foo']).run
+    end
+
+    def test_precompile_gemfile
+      assert_equal 0, CLI.new(['precompile', '--gemfile']).run
+    end
+  end
+end

test/test_helper.rb

@@ -71,6 +71,7 @@ def fnv1a_64(data)
         end
 
         def set_file(path, contents, mtime)
+          FileUtils.mkdir_p(File.dirname(path))
           File.write(path, contents)
           FileUtils.touch(path, mtime: mtime)
           path