Improve TokenUtils performance through caching

src/utils/TokenUtils.js

@@ -33,7 +33,54 @@
 define(function (require, exports, module) {
     "use strict";
 
-    var CodeMirror = require("thirdparty/CodeMirror2/lib/codemirror");
+    var _           = require("thirdparty/lodash"),
+        CodeMirror  = require("thirdparty/CodeMirror2/lib/codemirror");
+
+    var cache,
+        CACHE_MAX_AGE = 1000; // cache for 1 second
+
+
+    /*
+     * Caches the tokens for the given editor/line if needed
+     * @param {!CodeMirror} editor
+     * @param {!number} line
+     * @return {Array.<Object>} (Cached) array of tokens
+     */
+    function _manageCache(editor, line) {
+        if (!cache || !cache.tokens || cache.line !== line || cache.editor !== editor
+                || cache.timeStamp < Date.now() - CACHE_MAX_AGE) {
+            // Cache is outdated/no longer matching -> Update
+            var tokens = editor.getLineTokens(line, false);
+            // Add empty beginning-of-line token for backwards compatibility
+            tokens.unshift(editor.getTokenAt({line: line, ch: 0}, false));
+            cache = {
+                editor: editor,
+                line: line,
+                timeStamp: Date.now(),
+                tokens: tokens
+            };
+        }
+        return cache.tokens;
+    }
+
+    /*
+     * Like cm.getTokenAt, but with caching
+     * @param {!CodeMirror} editor
+     * @param {!{ch:number, line:number}} pos
+     * @param {boolean} precise If given, results in more current results. Suppresses caching.
+     * @return {Object} Token for position
+     */
+    function _getToken(editor, pos, precise) {
+        if (precise) {
+            cache = null; // reset cache
+            return editor.getTokenAt(pos, precise);
+        }
+        var cachedTokens = _manageCache(editor, pos.line),
+            token = _.find(cachedTokens, function (token) {
+                return token.end >= pos.ch;
+            });
+        return token || editor.getTokenAt(pos, precise); // fall back to CMs getTokenAt, for example in an empty line
+    }
 
    /**
      * Creates a context object for the given editor and position, suitable for passing to the
@@ -72,7 +119,7 @@ define(function (require, exports, module) {
         } else {
             ctx.pos.ch = ctx.token.start;
         }
-        ctx.token = ctx.editor.getTokenAt(ctx.pos, precise);
+        ctx.token = _getToken(ctx.editor, ctx.pos, precise);
         return true;
     }
 
@@ -107,7 +154,7 @@ define(function (require, exports, module) {
         } else {
             ctx.pos.ch = ctx.token.end + 1;
         }
-        ctx.token = ctx.editor.getTokenAt(ctx.pos, precise);
+        ctx.token = _getToken(ctx.editor, ctx.pos, precise);
         return true;
     }