rework record logic

.travis.yml

@@ -18,5 +18,11 @@ os:
   - linux
   - osx
 env:
-  - LISTEN_TESTS_DEFAULT_LAG=0.8 LISTEN_TESTS_DEFAULT_TCP_LAG=1.2
+  global:
+    - LISTEN_TESTS_DEFAULT_LAG=0.8 LISTEN_TESTS_DEFAULT_TCP_LAG=1.2
+  matrix:
+    - LISTEN_GEM_FSEVENT_NO_RECURSION=0 LISTEN_GEM_SIMULATE_FSEVENT=0
+    # try without recursion on both OSX and Linux (simulated OSX)
+    - LISTEN_GEM_FSEVENT_NO_RECURSION=1 LISTEN_GEM_SIMULATE_FSEVENT=1
+    - LISTEN_GEM_FSEVENT_NO_RECURSION=1 LISTEN_GEM_SIMULATE_FSEVENT=1 LISTEN_TESTS_POLLING=1
 sudo: false

Gemfile

@@ -16,7 +16,7 @@ group :test do
   gem 'celluloid', github: 'celluloid/celluloid', branch: '0-16-stable'
   gem 'celluloid-io', '>= 0.15.0'
   gem 'rake'
-  gem 'rspec', '~> 3.0.0rc1'
+  gem 'rspec', '~> 3.2'
   gem 'rspec-retry'
   gem 'coveralls'
 end

lib/listen/adapter.rb

@@ -4,10 +4,11 @@
 require 'listen/adapter/linux'
 require 'listen/adapter/polling'
 require 'listen/adapter/windows'
+require 'listen/adapter/simulated_darwin'
 
 module Listen
   module Adapter
-    OPTIMIZED_ADAPTERS = [Darwin, Linux, BSD, Windows]
+    OPTIMIZED_ADAPTERS = [Darwin, SimulatedDarwin, Linux, BSD, Windows]
     POLLING_FALLBACK_MESSAGE = 'Listen will be polling for changes.'\
       'Learn more at https://github.com/guard/listen#listen-adapters.'
 

lib/listen/adapter/darwin.rb

@@ -14,6 +14,10 @@ class Darwin < Base
 
       # NOTE: each directory gets a DIFFERENT callback!
       def _configure(dir, &callback)
+        if /1|true/ =~ ENV['LISTEN_GEM_FSEVENT_NO_RECURSION']
+          STDERR.puts "WARNING: Recursive scanning is disabled, which should"\
+            " be faster, but not all changes may be properly detected yet."
+        end
         require 'rb-fsevent'
         opts = { latency: options.latency }
 
@@ -45,7 +49,14 @@ def _process_event(dir, event)
           _log :debug, "fsevent: #{new_path}"
           # TODO: does this preserve symlinks?
           rel_path = new_path.relative_path_from(dir).to_s
-          _queue_change(:dir, dir, rel_path, recursive: true)
+
+          options =
+            if /1|true/ =~ ENV['LISTEN_GEM_FSEVENT_NO_RECURSION']
+              {}
+            else
+              {recursion: true}
+            end
+          _queue_change(:dir, dir, rel_path, options)
         end
       end
     end

lib/listen/adapter/linux.rb

@@ -49,16 +49,6 @@ def _process_event(dir, event)
 
         _log :debug, "inotify: #{rel_path} (#{event.flags.inspect})"
 
-        if /1|true/ =~ ENV['LISTEN_GEM_SIMULATE_FSEVENT']
-          if (event.flags & [:moved_to, :moved_from]) || _dir_event?(event)
-            rel_path = path.dirname.relative_path_from(dir).to_s
-            _queue_change(:dir, dir, rel_path, {})
-          else
-            _queue_change(:dir, dir, rel_path, {})
-          end
-          return
-        end
-
         return if _skip_event?(event)
 
         cookie_params = event.cookie.zero? ? {} : { cookie: event.cookie }

lib/listen/adapter/simulated_darwin.rb

@@ -0,0 +1,65 @@
+module Listen
+  module Adapter
+    class SimulatedDarwin < Linux
+      def self.usable?
+        os = RbConfig::CONFIG['target_os']
+        return false unless const_get('OS_REGEXP') =~ os
+        /1|true/ =~ ENV['LISTEN_GEM_SIMULATE_FSEVENT']
+      end
+
+      class FakeEvent
+        attr_reader :dir
+
+        def initialize(watched_dir, event)
+          # NOTE: avoid using event.absolute_name since new API
+          # will need to have a custom recursion implemented
+          # to properly match events to configured directories
+          @real_path = full_path(event).relative_path_from(watched_dir)
+          @dir = "#{Pathname(watched_dir) + dir_for_event(event, @real_path)}/"
+        end
+
+        def real_path
+          @real_path.to_s
+        end
+
+        private
+
+        def dir?(event)
+          event.flags.include?(:isdir)
+        end
+
+        def moved?(event)
+          (event.flags & [:moved_to, :moved_from])
+        end
+
+        def dir_for_event(event, rel_path)
+          (moved?(event) || dir?(event)) ?  rel_path.dirname : rel_path
+        end
+
+        def full_path(event)
+          Pathname.new(event.watcher.path) + event.name
+        end
+      end
+
+      private
+
+      def _process_event(watched_dir, event)
+        ev = FakeEvent.new(watched_dir, event)
+
+        _log(
+          :debug,
+          "fake_fsevent: #{ev.dir}(#{ev.real_path}=#{event.flags.inspect})")
+
+        _darwin.send(:_process_event, watched_dir, [ev.dir])
+      end
+
+      def _darwin
+        @darwin ||= Class.new(Darwin) do
+          def _configure(*_args)
+            # Skip FSEvent setup
+          end
+        end.new(mq: @mq)
+      end
+    end
+  end
+end

lib/listen/directory.rb

@@ -7,8 +7,6 @@ def self.scan(queue, sync_record, dir, rel_path, options)
 
       previous = sync_record.dir_entries(dir, rel_path)
 
-      record.add_dir(dir, rel_path)
-
       # TODO: use children(with_directory: false)
       path = dir + rel_path
       current = Set.new(path.children)

lib/listen/queue_optimizer.rb

@@ -77,7 +77,7 @@ def _reinterpret_related_changes(cookies)
           [[:modified, to_dir, to_file]]
         else
           not_silenced = changes.reject do |type, _, _, path, _|
-            _silenced?(Pathname(path), type)
+            _silenced?(Pathname.new(path), type)
           end
           not_silenced.map do |_, change, dir, path, _|
             [table.fetch(change, change), dir, path]

lib/listen/record.rb

@@ -11,12 +11,7 @@ class Record
 
     def initialize(listener)
       @listener = listener
-      @paths    = _auto_hash
-    end
-
-    def add_dir(dir, rel_path)
-      return if [nil, '', '.'].include? rel_path
-      @paths[dir.to_s][rel_path] ||= {}
+      @paths = {}
     end
 
     def update_file(dir, rel_path, data)
@@ -32,35 +27,22 @@ def unset_path(dir, rel_path)
     def file_data(dir, rel_path)
       root = @paths[dir.to_s]
       dirname, basename = Pathname(rel_path).split.map(&:to_s)
-      if [nil, '', '.'].include? dirname
-        root[basename] ||= {}
-        root[basename].dup
-      else
-        root[dirname] ||= {}
-        root[dirname][basename] ||= {}
-        root[dirname][basename].dup
-      end
+      dirname = '.' if [nil, '', '.'].include? dirname
+      fail "directory not watched: #{dir}" unless root
+
+      root[dirname] ||= {}
+      root[dirname][basename] ||= {}
+      root[dirname][basename].dup
     end
 
     def dir_entries(dir, rel_path)
-      tree = if [nil, '', '.'].include? rel_path.to_s
-               @paths[dir.to_s]
-             else
-               @paths[dir.to_s][rel_path.to_s] ||= _auto_hash
-               @paths[dir.to_s][rel_path.to_s]
-             end
-
-      result = {}
-      tree.each do |key, values|
-        # only get data for file entries
-        result[key] = values.key?(:mtime) ? values : {}
-      end
-      result
+      rel_path = '.' if [nil, '', '.'].include? rel_path.to_s
+      @paths[dir.to_s][rel_path.to_s] ||= {}
     end
 
     def build
       start = Time.now.to_f
-      @paths = _auto_hash
+      @paths = {}
 
       # TODO: refactor this out (1 Record = 1 watched dir)
       listener.directories.each do |directory|
@@ -75,39 +57,50 @@ def build
 
     private
 
-    def _auto_hash
-      Hash.new { |h, k| h[k] = Hash.new }
+    # TODO: refactor/refactor out
+    def add_dir(dir, rel_path)
+      rel_path = '.' if [nil, '', '.'].include? rel_path
+      dirname, basename = Pathname(rel_path).split.map(&:to_s)
+      basename = '.' if [nil, '', '.'].include? basename
+      root = (@paths[dir.to_s] ||= {})
+      if [nil, '', '.'].include?(dirname)
+        entries = (root['.'] || {})
+        entries.merge!(basename => {}) if basename != '.'
+        root['.'] = entries
+      else
+        root[rel_path] ||= {}
+      end
     end
 
     def _fast_update_file(dir, dirname, basename, data)
       root = @paths[dir.to_s]
-      if [nil, '', '.'].include? dirname
-        root[basename] = (root[basename] || {}).merge(data)
-      else
-        root[dirname] ||= {}
-        root[dirname][basename] = (root[dirname][basename] || {}).merge(data)
+      dirname = '.' if [nil, '', '.'].include? dirname
+
+      internal_dirname, internal_key = ::File.split(dirname.to_s)
+      if internal_dirname == '.' && internal_key != '.'
+        root[internal_dirname] ||= {}
+        root[internal_dirname][internal_key] ||= {}
       end
+
+      root[dirname] ||= {}
+      root[dirname][basename] = (root[dirname][basename] || {}).merge(data)
     end
 
     def _fast_unset_path(dir, dirname, basename)
       root = @paths[dir.to_s]
       # this may need to be reworked to properly remove
       # entries from a tree, without adding non-existing dirs to the record
-      if [nil, '', '.'].include? dirname
-        return unless root.key?(basename)
-        root.delete(basename)
-      else
-        return unless root.key?(dirname)
-        root[dirname].delete(basename)
-      end
+      dirname = '.' if [nil, '', '.'].include? dirname
+      return unless root.key?(dirname)
+      root[dirname].delete(basename)
     end
 
     # TODO: test with a file name given
     # TODO: test other permissions
     # TODO: test with mixed encoding
     def _fast_build(root)
       symlink_detector = SymlinkDetector.new
-      @paths[root] = _auto_hash
+      @paths[root] = {}
       remaining = Queue.new
       remaining << Entry.new(root, nil, nil)
       _fast_build_dir(remaining, symlink_detector) until remaining.empty?