Add BalanceHeuristicExpand, 1.7x faster search.

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## HEAD (unreleased)
 
+## 4.0.0
+
+- Introduce experimental algorithm for block expansion, 1.7x faster overall search (https://github.com/zombocom/dead_end/pull/129)
+
 ## 3.1.1
 
 - Fix case where Ripper lexing identified incorrect code as a keyword (https://github.com/zombocom/dead_end/pull/122)

Gemfile.lock

@@ -45,7 +45,7 @@ GEM
       rubocop-ast (>= 0.4.0)
     ruby-prof (1.4.3)
     ruby-progressbar (1.11.0)
-    stackprof (0.2.16)
+    stackprof (0.2.17)
     standard (1.3.0)
       rubocop (= 1.20.0)
       rubocop-performance (= 1.11.5)

lib/dead_end/api.rb

@@ -16,7 +16,7 @@ class Error < StandardError; end
 
   # DeadEnd.handle_error [Public]
   #
-  # Takes a `SyntaxError`` exception, uses the
+  # Takes a `SyntaxError` exception, uses the
   # error message to locate the file. Then the file
   # will be analyzed to find the location of the syntax
   # error and emit that location to stderr.
@@ -187,12 +187,14 @@ def self.valid?(source)
 require_relative "lex_all"
 require_relative "code_line"
 require_relative "code_block"
-require_relative "block_expand"
+require_relative "lex_pair_diff"
 require_relative "ripper_errors"
 require_relative "priority_queue"
 require_relative "unvisited_lines"
 require_relative "around_block_scan"
+require_relative "indent_block_expand"
 require_relative "priority_engulf_queue"
 require_relative "pathname_from_message"
 require_relative "display_invalid_blocks"
+require_relative "balance_heuristic_expand"
 require_relative "parse_blocks_from_indent_line"

lib/dead_end/balance_heuristic_expand.rb

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+
+module DeadEnd
+  # Expand code based on lexical heuristic
+  #
+  # Code that has unbalanced pairs cannot be valid
+  # i.e. `{` must always be matched with a `}`.
+  #
+  # This expansion class exploits that knowledge to
+  # expand a logical block towards equal pairs.
+  #
+  # For example: if code is missing a `]` it cannot
+  # be on a line above, so it must expand down
+  #
+  # This heuristic allows us to make larger and more
+  # accurate expansions which means fewer invalid
+  # blocks to check which means overall faster search.
+  #
+  # This class depends on another class LexPairDiff can be
+  # accesssed per-line. It holds the delta of tracked directional
+  # pairs: curly brackets, square brackets, parens, and kw/end
+  # with positive count (leaning left), 0 (balanced), or negative
+  # count (leaning right).
+  #
+  # With this lexical diff information we can look around a given
+  # block and move with inteligently. For instance if the current
+  # block has a miss matched `end` and the line above it holds
+  # `def foo` then the block will be expanded up to capture that line.
+  #
+  # An unbalanced block can never be valid (this provides info to
+  # the overall search). However a balanced block may contain other syntax
+  # error and so must be re-checked using Ripper (slow).
+  #
+  # Example
+  #
+  #   lines = CodeLines.from_source(<~'EOM')
+  #     if bark?
+  #     end
+  #   EOM
+  #   block = CodeBlock.new(lines: lines[0])
+  #
+  #   expand = BalanceHeuristicExpand.new(
+  #     code_lines: lines,
+  #     block: block
+  #   )
+  #   expand.direction # => :down
+  #   expand.call
+  #   expand.direction # => :equal
+  #
+  #   expect(expand.to_s).to eq(lines.join)
+  class BalanceHeuristicExpand
+    attr_reader :start_index, :end_index
+
+    def initialize(code_lines:, block:)
+      @block = block
+      @iterations = 0
+      @code_lines = code_lines
+      @last_index = @code_lines.length - 1
+      @max_iterations = @code_lines.length * 2
+      @start_index = block.lines.first.index
+      @end_index = block.lines.last.index
+      @last_equal_range = nil
+
+      set_lex_diff_from(block)
+    end
+
+    private def set_lex_diff_from(block)
+      @lex_diff = LexPairDiff.new(
+        curly: 0,
+        square: 0,
+        parens: 0,
+        kw_end: 0
+      )
+      block.lines.each do |line|
+        @lex_diff.concat(line.lex_diff)
+      end
+    end
+
+    # Converts the searched lines into a source string
+    def to_s
+      @code_lines[start_index..end_index].join
+    end
+
+    # Converts the searched lines into a code block
+    def to_block
+      CodeBlock.new(lines: @code_lines[start_index..end_index])
+    end
+
+    # Returns true if all lines are equal
+    def balanced?
+      @lex_diff.balanced?
+    end
+
+    # Returns false if captured lines are "leaning"
+    # one direction
+    def unbalanced?
+      !balanced?
+    end
+
+    # Main search entrypoint
+    #
+    # Essentially a state machine, determine the leaning
+    # of the given block, then figure out how to either
+    # move it towards balanced, or expand it while keeping
+    # it balanced.
+    def call
+      case direction
+      when :up
+        # the goal is to become balanced
+        while keep_going? && direction == :up && try_expand_up
+        end
+      when :down
+        # the goal is to become balanced
+        while keep_going? && direction == :down && try_expand_down
+        end
+      when :equal
+        while keep_going? && grab_equal_or {
+          # Cannot create a balanced expansion, choose to be unbalanced
+          try_expand_up
+        }
+        end
+
+        call # Recurse
+      when :both
+        while keep_going? && grab_equal_or {
+          try_expand_up
+          try_expand_down
+        }
+        end
+      when :stop
+        return self
+      end
+
+      self
+    end
+
+    # Convert a lex diff to a direction to search
+    #
+    # leaning left -> down
+    # leaning right -> up
+    #
+    def direction
+      leaning = @lex_diff.leaning
+      case leaning
+      when :left # go down
+        stop_bottom? ? :stop : :down
+      when :right # go up
+        stop_top? ? :stop : :up
+      when :equal, :both
+        if stop_top? && stop_bottom?
+          :stop
+        elsif stop_top? && !stop_bottom?
+          :down
+        elsif !stop_top? && stop_bottom?
+          :up
+        else
+          leaning
+        end
+      end
+    end
+
+    # Limit rspec failure output
+    def inspect
+      "#<DeadEnd::BalanceHeuristicExpand:0x0000000115lol too big>"
+    end
+
+    # Upper bound on iterations
+    private def keep_going?
+      if @iterations < @max_iterations
+        @iterations += 1
+        true
+      else
+        warn <<~EOM
+          DeadEnd: Internal problem detected, possible infinite loop in #{self.class}
+
+          Please open a ticket with the following information. Max: #{@max_iterations}, actual: #{@iterations}
+
+          Original block:
+
+          ```
+          #{@block.lines.map(&:original).join}```
+
+          Stuck at:
+
+          ```
+          #{to_block.lines.map(&:original).join}```
+        EOM
+
+        false
+      end
+    end
+
+    # Attempt to grab "free" lines
+    #
+    # if either above, below or both are
+    # balanced, take them, return true.
+    #
+    # If above is leaning left and below
+    # is leaning right and they cancel out
+    # take them, return true.
+    #
+    # If we couldn't grab any balanced lines
+    # then call the block and return false.
+    private def grab_equal_or
+      did_expand = false
+      if above&.balanced?
+        did_expand = true
+        try_expand_up
+      end
+
+      if below&.balanced?
+        did_expand = true
+        try_expand_down
+      end
+
+      return true if did_expand
+
+      if make_balanced_from_up_down?
+        try_expand_up
+        try_expand_down
+        true
+      else
+        yield
+        false
+      end
+    end
+
+    # If up is leaning left and down is leaning right
+    # they might cancel out, to make a complete
+    # and balanced block
+    private def make_balanced_from_up_down?
+      return false if above.nil? || below.nil?
+      return false if above.lex_diff.leaning != :left
+      return false if below.lex_diff.leaning != :right
+
+      @lex_diff.dup.concat(above.lex_diff).concat(below.lex_diff).balanced?
+    end
+
+    # The line above the current location
+    private def above
+      @code_lines[@start_index - 1] unless stop_top?
+    end
+
+    # The line below the current location
+    private def below
+      @code_lines[@end_index + 1] unless stop_bottom?
+    end
+
+    # Mutates the start index and applies the new line's
+    # lex diff
+    private def expand_up
+      @start_index -= 1
+      @lex_diff.concat(@code_lines[@start_index].lex_diff)
+    end
+
+    private def try_expand_up
+      stop_top? ? false : expand_up
+    end
+
+    private def try_expand_down
+      stop_bottom? ? false : expand_down
+    end
+
+    # Mutates the end index and applies the new line's
+    # lex diff
+    private def expand_down
+      @end_index += 1
+      @lex_diff.concat(@code_lines[@end_index].lex_diff)
+    end
+
+    # Returns true when we can no longer expand up
+    private def stop_top?
+      @start_index == 0
+    end
+
+    # Returns true when we can no longer expand down
+    private def stop_bottom?
+      @end_index == @last_index
+    end
+  end
+end

lib/dead_end/code_frontier.rb

@@ -50,6 +50,8 @@ module DeadEnd
   #   CodeFrontier#detect_invalid_blocks
   #
   class CodeFrontier
+    attr_reader :queue
+
     def initialize(code_lines:, unvisited: UnvisitedLines.new(code_lines: code_lines))
       @code_lines = code_lines
       @unvisited = unvisited

lib/dead_end/code_line.rb

@@ -38,7 +38,7 @@ def self.from_source(source, lines: nil)
       end
     end
 
-    attr_reader :line, :index, :lex, :line_number, :indent
+    attr_reader :line, :index, :lex, :line_number, :indent, :lex_diff
     def initialize(line:, index:, lex:)
       @lex = lex
       @line = line
@@ -57,6 +57,16 @@ def initialize(line:, index:, lex:)
       end
 
       set_kw_end
+
+      @lex_diff = LexPairDiff.from_lex(
+        lex: @lex,
+        is_kw: is_kw?,
+        is_end: is_end?
+      )
+    end
+
+    def balanced?
+      @lex_diff.balanced?
     end
 
     # Used for stable sort via indentation level