Merge pull request #612 from Shopify/pm/lazy-loadable-backend

LazyLoadable Backend

lib/i18n/backend.rb

@@ -3,15 +3,16 @@
 module I18n
   module Backend
     autoload :Base,                  'i18n/backend/base'
-    autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler'
     autoload :Cache,                 'i18n/backend/cache'
     autoload :CacheFile,             'i18n/backend/cache_file'
     autoload :Cascade,               'i18n/backend/cascade'
     autoload :Chain,                 'i18n/backend/chain'
     autoload :Fallbacks,             'i18n/backend/fallbacks'
     autoload :Flatten,               'i18n/backend/flatten'
     autoload :Gettext,               'i18n/backend/gettext'
+    autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler'
     autoload :KeyValue,              'i18n/backend/key_value'
+    autoload :LazyLoadable,          'i18n/backend/lazy_loadable'
     autoload :Memoize,               'i18n/backend/memoize'
     autoload :Metadata,              'i18n/backend/metadata'
     autoload :Pluralization,         'i18n/backend/pluralization'

lib/i18n/backend/base.rb

@@ -13,7 +13,10 @@ module Base
       # for details.
       def load_translations(*filenames)
         filenames = I18n.load_path if filenames.empty?
-        filenames.flatten.each { |filename| load_file(filename) }
+        filenames.flatten.each do |filename|
+          loaded_translations = load_file(filename)
+          yield filename, loaded_translations if block_given?
+        end
       end
 
       # This method receives a locale, a data hash and options for storing translations.
@@ -226,6 +229,8 @@ def load_file(filename)
             raise InvalidLocaleData.new(filename, 'expects it to return a hash, but does not')
           end
           data.each { |locale, d| store_translations(locale, d || {}, skip_symbolize_keys: keys_symbolized) }
+
+          data
         end
 
         # Loads a plain Ruby translations file. eval'ing the file must yield

lib/i18n/backend/lazy_loadable.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module I18n
+  module Backend
+    # Backend that lazy loads translations based on the current locale. This
+    # implementation avoids loading all translations up front. Instead, it only
+    # loads the translations that belong to the current locale. This offers a
+    # performance incentive in local development and test environments for
+    # applications with many translations for many different locales. It's
+    # particularly useful when the application only refers to a single locales'
+    # translations at a time (ex. A Rails workload).  The implementation
+    # identifies which translation files from the load path belong to the
+    # current locale by pattern matching against their path name.
+    #
+    # Specifically, a translation file is considered to belong to a locale if:
+    # a) the filename is in the I18n load path
+    # b) the filename ends in a supported extension (ie. .yml, .json, .po, .rb)
+    # c) the filename starts with the locale identifier
+    # d) the locale identifier and optional proceeding text is separated by an underscore, ie. "_".
+    #
+    # Examples:
+    # Valid files that will be selected by this backend:
+    #
+    # "files/locales/en_translation.yml" (Selected for locale "en")
+    # "files/locales/fr.po"  (Selected for locale "fr")
+    #
+    # Invalid files that won't be selected by this backend:
+    #
+    # "files/locales/translation-file"
+    # "files/locales/en-translation.unsupported"
+    # "files/locales/french/translation.yml"
+    # "files/locales/fr/translation.yml"
+    #
+    # The implementation uses this assumption to defer the loading of
+    # translation files until the current locale actually requires them.
+    #
+    # The backend has two working modes: lazy_load and eager_load.
+    #
+    # Note: This backend should only be enabled in test environments!
+    # When the mode is set to false, the backend behaves exactly like the
+    # Simple backend, with an additional check that the paths being loaded
+    # abide by the format. If paths can't be matched to the format, an error is raised.
+    #
+    # You can configure lazy loaded backends through the initializer or backends
+    # accessor:
+    #
+    #   # In test environments
+    #
+    #   I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: true)
+    #
+    #   # In other environments, such as production and CI
+    #
+    #   I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: false) # default
+    #
+    class LocaleExtractor
+      class << self
+        def locale_from_path(path)
+          name = File.basename(path, ".*")
+          locale = name.split("_").first
+          locale.to_sym unless locale.nil?
+        end
+      end
+    end
+
+    class LazyLoadable < Simple
+      def initialize(lazy_load: false)
+        @lazy_load = lazy_load
+      end
+
+      # Returns whether the current locale is initialized.
+      def initialized?
+        if lazy_load?
+          initialized_locales[I18n.locale]
+        else
+          super
+        end
+      end
+
+      # Clean up translations and uninitialize all locales.
+      def reload!
+        if lazy_load?
+          @initialized_locales = nil
+          @translations = nil
+        else
+          super
+        end
+      end
+
+      # Eager loading is not supported in the lazy context.
+      def eager_load!
+        if lazy_load?
+          raise UnsupportedMethod.new(__method__, self.class, "Cannot eager load translations because backend was configured with lazy_load: true.")
+        else
+          super
+        end
+      end
+
+      # Parse the load path and extract all locales.
+      def available_locales
+        if lazy_load?
+          I18n.load_path.map { |path| LocaleExtractor.locale_from_path(path) }
+        else
+          super
+        end
+      end
+
+      def lookup(locale, key, scope = [], options = EMPTY_HASH)
+        if lazy_load?
+          I18n.with_locale(locale) do
+            super
+          end
+        else
+          super
+        end
+      end
+
+      protected
+
+
+      # Load translations from files that belong to the current locale.
+      def init_translations
+        file_errors = if lazy_load?
+          initialized_locales[I18n.locale] = true
+          load_translations_and_collect_file_errors(filenames_for_current_locale)
+        else
+          @initialized = true
+          load_translations_and_collect_file_errors(I18n.load_path)
+        end
+
+        raise InvalidFilenames.new(file_errors) unless file_errors.empty?
+      end
+
+      def initialized_locales
+        @initialized_locales ||= Hash.new(false)
+      end
+
+      private
+
+      def lazy_load?
+        @lazy_load
+      end
+
+      class FilenameIncorrect < StandardError
+        def initialize(file, expected_locale, unexpected_locales)
+          super "#{file} can only load translations for \"#{expected_locale}\". Found translations for: #{unexpected_locales}."
+        end
+      end
+
+      # Loads each file supplied and asserts that the file only loads
+      # translations as expected by the name. The method returns a list of
+      # errors corresponding to offending files.
+      def load_translations_and_collect_file_errors(files)
+        errors = []
+
+        load_translations(files) do |file, loaded_translations|
+          assert_file_named_correctly!(file, loaded_translations)
+        rescue FilenameIncorrect => e
+          errors << e
+        end
+
+        errors
+      end
+
+      # Select all files from I18n load path that belong to current locale.
+      # These files must start with the locale identifier (ie. "en", "pt-BR"),
+      # followed by an "_" demarcation to separate proceeding text.
+      def filenames_for_current_locale
+        I18n.load_path.flatten.select do |path|
+          LocaleExtractor.locale_from_path(path) == I18n.locale
+        end
+      end
+
+      # Checks if a filename is named in correspondence to the translations it loaded.
+      # The locale extracted from the path must be the single locale loaded in the translations.
+      def assert_file_named_correctly!(file, translations)
+        loaded_locales = translations.keys.map(&:to_sym)
+        expected_locale = LocaleExtractor.locale_from_path(file)
+        unexpected_locales = loaded_locales.reject { |locale| locale == expected_locale }
+
+        raise FilenameIncorrect.new(file, expected_locale, unexpected_locales) unless unexpected_locales.empty?
+      end
+    end
+  end
+end

lib/i18n/exceptions.rb

@@ -110,4 +110,38 @@ def initialize(type, filename)
       super "can not load translations from #{filename}, the file type #{type} is not known"
     end
   end
+
+  class UnsupportedMethod < ArgumentError
+    attr_reader :method, :backend_klass, :msg
+    def initialize(method, backend_klass, msg)
+      @method = method
+      @backend_klass = backend_klass
+      @msg = msg
+      super "#{backend_klass} does not support the ##{method} method. #{msg}"
+    end
+  end
+
+  class InvalidFilenames < ArgumentError
+    NUMBER_OF_ERRORS_SHOWN = 20
+    def initialize(file_errors)
+      super <<~MSG
+        Found #{file_errors.count} error(s).
+        The first #{[file_errors.count, NUMBER_OF_ERRORS_SHOWN].min} error(s):
+        #{file_errors.map(&:message).first(NUMBER_OF_ERRORS_SHOWN).join("\n")}
+
+        To use the LazyLoadable backend:
+        1. Filenames must start with the locale.
+        2. An underscore must separate the locale with any optional text that follows.
+        3. The file must only contain translation data for the single locale.
+
+        Example:
+        "/config/locales/fr.yml" which contains:
+        ```yml
+          fr:
+            dog:
+              chien
+        ```
+      MSG
+    end
+  end
 end

lib/i18n/tests/basics.rb

@@ -5,12 +5,11 @@ def teardown
         I18n.available_locales = nil
       end
 
-      test "available_locales returns the locales stored to the backend by default" do
+      test "available_locales returns the available_locales produced by the backend, by default" do
         I18n.backend.store_translations('de', :foo => 'bar')
         I18n.backend.store_translations('en', :foo => 'foo')
 
-        assert I18n.available_locales.include?(:de)
-        assert I18n.available_locales.include?(:en)
+        assert_equal I18n.available_locales, I18n.backend.available_locales
       end
 
       test "available_locales can be set to something else independently from the actual locale data" do
@@ -24,8 +23,7 @@ def teardown
         assert_equal [:foo, :bar], I18n.available_locales
 
         I18n.available_locales = nil
-        assert I18n.available_locales.include?(:de)
-        assert I18n.available_locales.include?(:en)
+        assert_equal I18n.available_locales, I18n.backend.available_locales
       end
 
       test "available_locales memoizes when set explicitely" do

test/api/lazy_loadable_test.rb

@@ -0,0 +1,24 @@
+require 'test_helper'
+
+class I18nLazyLoadableBackendApiTest < I18n::TestCase
+  def setup
+    I18n.backend = I18n::Backend::LazyLoadable.new
+    super
+  end
+
+  include I18n::Tests::Basics
+  include I18n::Tests::Defaults
+  include I18n::Tests::Interpolation
+  include I18n::Tests::Link
+  include I18n::Tests::Lookup
+  include I18n::Tests::Pluralization
+  include I18n::Tests::Procs
+  include I18n::Tests::Localization::Date
+  include I18n::Tests::Localization::DateTime
+  include I18n::Tests::Localization::Time
+  include I18n::Tests::Localization::Procs
+
+  test "make sure we use the LazyLoadable backend" do
+    assert_equal I18n::Backend::LazyLoadable, I18n.backend.class
+  end
+end