Improve TokenUtils performance through caching

src/language/HTMLUtils.js

@@ -508,19 +508,16 @@ define(function (require, exports, module) {
      */
     function findBlocks(editor, modeName) {
         // Start scanning from beginning of file
-        var blocks = [],
-            cm = editor._codeMirror,
+        var ctx = TokenUtils.getInitialContext(editor._codeMirror, {line: 0, ch: 0}),
+            blocks = [],
             currentBlock = null,
             inBlock = false,
-            line,
-            lineCount = editor.lineCount(),
-            outerMode = cm.getMode(),
-            previousMode,
+            outerMode = editor._codeMirror.getMode(),
             tokenModeName,
-            tokens;
+            previousMode;
 
-        function checkForBlock(ctx) {
-            tokenModeName = CodeMirror.innerMode(outerMode, ctx.state).mode.name;
+        while (TokenUtils.moveNextToken(ctx, false)) {
+            tokenModeName = CodeMirror.innerMode(outerMode, ctx.token.state).mode.name;
             if (inBlock) {
                 if (!currentBlock.end) {
                     // Handle empty blocks
@@ -532,13 +529,13 @@ define(function (require, exports, module) {
                     currentBlock.text = editor.document.getRange(currentBlock.start, currentBlock.end);
                     inBlock = false;
                 } else {
-                    currentBlock.end = { line: line, ch: ctx.end };
+                    currentBlock.end = { line: ctx.pos.line, ch: ctx.pos.ch };
                 }
             } else {
                 // Check for start of a block
                 if (tokenModeName === modeName) {
                     currentBlock = {
-                        start: { line: line, ch: ctx.end }
+                        start: { line: ctx.pos.line, ch: ctx.pos.ch }
                     };
                     blocks.push(currentBlock);
                     inBlock = true;
@@ -549,12 +546,6 @@ define(function (require, exports, module) {
             }
         }
 
-        for (line = 0; line < lineCount; line++) {
-            tokens = cm.getLineTokens(line);
-            tokens.forEach(checkForBlock);
-            tokens = null; // Garbage collection - tokens can be pretty big
-        }
-
         return blocks;
     }
 

src/utils/TokenUtils.js

@@ -33,20 +33,73 @@
 define(function (require, exports, module) {
     "use strict";
 
-    var CodeMirror = require("thirdparty/CodeMirror2/lib/codemirror");
+    var _           = require("thirdparty/lodash"),
+        CodeMirror  = require("thirdparty/CodeMirror2/lib/codemirror");
+
+    var cache;
+
+
+    function _clearCache(cm) {
+        cache = null;
+        if (cm) { // event handler
+            cm.off("changes", _clearCache);
+        }
+    }
+
+    /*
+     * Caches the tokens for the given editor/line if needed
+     * @param {!CodeMirror} cm
+     * @param {!number} line
+     * @return {Array.<Object>} (Cached) array of tokens
+     */
+    function _manageCache(cm, line) {
+        if (!cache || !cache.tokens || cache.line !== line || cache.cm !== cm) {
+            // Cache is no longer matching -> Update
+            var tokens = cm.getLineTokens(line, false);
+            // Add empty beginning-of-line token for backwards compatibility
+            tokens.unshift(cm.getTokenAt({line: line, ch: 0}, false));
+            cache = {
+                cm: cm,
+                line: line,
+                timeStamp: Date.now(),
+                tokens: tokens,
+            };
+            cm.off("changes", _clearCache);
+            cm.on("changes", _clearCache);
+        }
+        return cache.tokens;
+    }
+
+    /*
+     * Like cm.getTokenAt, but with caching
+     * @param {!CodeMirror} cm
+     * @param {!{ch:number, line:number}} pos
+     * @param {boolean} precise If given, results in more current results. Suppresses caching.
+     * @return {Object} Token for position
+     */
+    function _getToken(cm, pos, precise) {
+        if (precise) {
+            _clearCache(); // reset cache
+            return cm.getTokenAt(pos, precise);
+        }
+        var cachedTokens    = _manageCache(cm, pos.line),
+            tokenIndex      = _.sortedIndex(cachedTokens, {end: pos.ch}, "end"), // binary search is faster for long arrays
+            token           = cachedTokens[tokenIndex];
+        return token || cm.getTokenAt(pos, precise); // fall back to CMs getTokenAt, for example in an empty line
+    }
 
    /**
      * Creates a context object for the given editor and position, suitable for passing to the
      * move functions.
-     * @param {!CodeMirror} editor
+     * @param {!CodeMirror} cm
      * @param {!{ch:number, line:number}} pos
      * @return {!{editor:!CodeMirror, pos:!{ch:number, line:number}, token:Object}}
      */
-    function getInitialContext(editor, pos) {
+    function getInitialContext(cm, pos) {
         return {
-            "editor": editor,
+            "editor": cm,
             "pos": pos,
-            "token": editor.getTokenAt(pos, true)
+            "token": cm.getTokenAt(pos, true)
         };
     }
 
@@ -72,7 +125,7 @@ define(function (require, exports, module) {
         } else {
             ctx.pos.ch = ctx.token.start;
         }
-        ctx.token = ctx.editor.getTokenAt(ctx.pos, precise);
+        ctx.token = _getToken(ctx.editor, ctx.pos, precise);
         return true;
     }
 
@@ -107,7 +160,7 @@ define(function (require, exports, module) {
         } else {
             ctx.pos.ch = ctx.token.end + 1;
         }
-        ctx.token = ctx.editor.getTokenAt(ctx.pos, precise);
+        ctx.token = _getToken(ctx.editor, ctx.pos, precise);
         return true;
     }