fix: move right of cursor tokens to end of ghost text. (#5616)

* fix: move tokens to the right of ghost text down

src/autocomplete/inline_test.js

@@ -105,7 +105,7 @@ module.exports = {
         inline.show(editor, completions[3], "f");
         editor.renderer.$loop._flush();
         assert.strictEqual(getAllLines(), textBase + "function foo() {");
-        assert.strictEqual(editor.renderer.$ghostTextWidget.el.innerHTML, `<div>        console.log('test');</div><div>    }</div>`);
+        assert.strictEqual(editor.renderer.$ghostTextWidget.el.innerHTML, `<div><span class="ace_ghost_text">        console.log('test');</span></div><div><span class="ace_ghost_text">    }</span><span></span></div>`);
         done();
     },
     "test: boundary tests": function(done) {
@@ -314,7 +314,69 @@ module.exports = {
         inline.show(editor, completions[8], "f");
         editor.renderer.$loop._flush();
         assert.strictEqual(getAllLines(), textBase + "foo suggestion with a");
-        assert.strictEqual(editor.renderer.$ghostTextWidget.el.innerHTML, `<div> </div><div> </div><div>gap</div>`);
+        assert.strictEqual(editor.renderer.$ghostTextWidget.el.innerHTML, `<div><span class="ace_ghost_text"> </span></div><div><span class="ace_ghost_text"> </span></div><div><span class="ace_ghost_text">gap</span><span></span></div>`);
+        done();
+    },
+    "test: moves tokens to the right of cursor to the end of ghost text for multi line ghost text": function(done) {
+        editor.execCommand("removetolinestarthard");
+        editor.execCommand("insertstring", "f hi I should be hidden");
+        editor.execCommand("gotolinestart");
+        editor.execCommand("gotoright");
+        editor.renderer.$loop._flush();
+        assert.equal(editor.renderer.$ghostTextWidget, null);
+        inline.show(editor, completions[8], "f");
+        editor.renderer.$loop._flush();
+        assert.strictEqual(getAllLines(), textBase.replaceAll(" ", "") + "foo suggestion with a hi I should be hidden");
+
+        // The string to the right of the cursor should be hidden tokens now.
+        var tokens = editor.session.getTokens(2);
+        assert.strictEqual(tokens[2].value, " hi I should be hidden");
+        assert.strictEqual(tokens[2].type, "hidden_token");
+
+        // And should be added to the ghost text widget.
+        assert.strictEqual(editor.renderer.$ghostTextWidget.el.textContent, "  gap hi I should be hidden");
+
+        // Hide inline
+        inline.hide();
+        editor.renderer.$loop._flush();
+        assert.equal(editor.renderer.$ghostTextWidget, null);
+
+        // Text to the right of the cursor should be tokenized normally again.
+        var tokens = editor.session.getTokens(2);
+        assert.strictEqual(tokens[0].value, "f hi I should be hidden");
+        assert.strictEqual(tokens[0].type, "text");
+
+        done();
+    },
+    "test: moves tokens to the right of cursor to the end of ghost text for multi line ghost text when triggered inside token": function(done) {
+        editor.execCommand("removetolinestarthard");
+        editor.execCommand("insertstring", "fhi I should be hidden");
+        editor.execCommand("gotolinestart");
+        editor.execCommand("gotoright");
+        editor.renderer.$loop._flush();
+        assert.equal(editor.renderer.$ghostTextWidget, null);
+        inline.show(editor, completions[8], "f");
+        editor.renderer.$loop._flush();
+        assert.strictEqual(getAllLines(), textBase.replaceAll(" ", "") + "foo suggestion with ahi I should be hidden");
+
+        // The string to the right of the cursor should be hidden tokens now.
+        var tokens = editor.session.getTokens(2);
+        assert.strictEqual(tokens[2].value, "hi I should be hidden");
+        assert.strictEqual(tokens[2].type, "hidden_token");
+
+        // And should be added to the ghost text widget.
+        assert.strictEqual(editor.renderer.$ghostTextWidget.el.textContent, "  gaphi I should be hidden");
+
+        // Hide inline
+        inline.hide();
+        editor.renderer.$loop._flush();
+        assert.equal(editor.renderer.$ghostTextWidget, null);
+
+        // Text to the right of the cursor should be tokenized normally again.
+        var tokens = editor.session.getTokens(2);
+        assert.strictEqual(tokens[0].value, "fhi I should be hidden");
+        assert.strictEqual(tokens[0].type, "text");
+
         done();
     },
     tearDown: function() {

src/css/editor-css.js

@@ -660,7 +660,7 @@ module.exports = `
     font-style: italic;
 }
 
-.ace_ghost_text > div {
+.ace_ghost_text_container > div {
     white-space: pre;
 }
 
@@ -680,4 +680,8 @@ module.exports = `
     width:1px;
     height:1px;
     overflow:hidden;
+}
+
+.ace_hidden_token {
+    display: none;
 }`;

src/ext/inline_autocomplete_test.js

@@ -358,7 +358,7 @@ module.exports = {
         typeAndChange("u", "n");
         editor.renderer.$loop._flush();
         assert.strictEqual(autocomplete.isOpen(), true);
-        assert.equal(getAllLines(), `function foo() {\n<div>    console.log('test');</div><div>}</div>`);
+        assert.equal(getAllLines(), `function foo() {\n<div><span class="ace_ghost_text">    console.log('test');</span></div><div><span class="ace_ghost_text">}</span><span></span></div>`);
 
         typeAndChange("d");
         editor.renderer.$loop._flush();

src/virtual_renderer.js

@@ -23,6 +23,7 @@ var editorCss = require("./css/editor-css");
 var Decorator = require("./layer/decorators").Decorator;
 
 var useragent = require("./lib/useragent");
+const isTextToken = require("./layer/text_util").isTextToken;
 
 dom.importCssString(editorCss, "ace_editor.css", false);
 
@@ -1779,8 +1780,14 @@ class VirtualRenderer {
 
         var widgetDiv = dom.createElement("div");
         if (textChunks.length > 1) {
+            // If there are tokens to the right of the cursor, hide those.
+            var hiddenTokens = this.hideTokensAfterPosition(insertPosition.row, insertPosition.column);
+
+            var lastLineDiv;
             textChunks.slice(1).forEach(el => {
                 var chunkDiv = dom.createElement("div");
+                var chunkSpan = dom.createElement("span");
+                chunkSpan.className = "ace_ghost_text";            
 
                 // If the line is wider than the viewport, wrap the line
                 if (el.wrapped) chunkDiv.className = "ghost_text_line_wrapped";
@@ -1789,15 +1796,28 @@ class VirtualRenderer {
                 // textcontent so that browsers render the empty line div.
                 if (el.text.length === 0) el.text = " "; 
 
-                chunkDiv.appendChild(dom.createTextNode(el.text));
+                chunkSpan.appendChild(dom.createTextNode(el.text));
+                chunkDiv.appendChild(chunkSpan);
                 widgetDiv.appendChild(chunkDiv);
+
+                // Overwrite lastLineDiv every iteration so at the end it points to 
+                // the last added element.
+                lastLineDiv = chunkDiv;
+            });
+
+            // Add the hidden tokens to the last line of the ghost text.
+            hiddenTokens.forEach(token => {
+                var element = dom.createElement("span");
+                if (!isTextToken(token.type)) element.className = "ace_" + token.type.replace(/\./g, " ace_");
+                element.appendChild(dom.createTextNode(token.value));
+                lastLineDiv.appendChild(element);
             });
 
             this.$ghostTextWidget = {
                 el: widgetDiv,
                 row: insertPosition.row,
                 column: insertPosition.column,
-                className: "ace_ghost_text"
+                className: "ace_ghost_text_container"
             };
             this.session.widgetManager.addLineWidget(this.$ghostTextWidget);
 
@@ -1902,6 +1922,35 @@ class VirtualRenderer {
         this.updateLines(row, row);
     }
 
+    // Hide all non-ghost-text tokens to the right of a given position.
+    hideTokensAfterPosition(row, column) {
+        var tokens = this.session.getTokens(row);
+        var l = 0;
+        var hasPassedCursor = false;
+        var hiddenTokens = [];
+        // Loop over all tokens and track at what position in the line they end.
+        for (var i = 0; i < tokens.length; i++) {
+            var token = tokens[i];
+            l += token.value.length;
+
+            if (token.type === "ghost_text") continue;
+
+            // If we've already passed the current cursor position, mark all of them as hidden.
+            if (hasPassedCursor) {
+                hiddenTokens.push({type: token.type, value: token.value});
+                token.type = "hidden_token";
+                continue;
+            }
+            // We call this method after we call addToken, so we are guaranteed a new token starts at the cursor position.
+            // Once we reached that point in the loop, flip the flag. 
+            if (l === column) {
+                hasPassedCursor = true;
+            } 
+        }
+        this.updateLines(row, row);
+        return hiddenTokens;
+    }
+
     removeExtraToken(row, column) {
         this.session.bgTokenizer.lines[row] = null;
         this.updateLines(row, row);

src/virtual_renderer_test.js

@@ -376,7 +376,7 @@ module.exports = {
         editor.renderer.$loop._flush();
         assert.equal(editor.renderer.content.textContent, "abcdefGhost1");
 
-        assert.equal(editor.session.lineWidgets[0].el.innerHTML, `<div>Ghost2</div><div>Ghost3</div>`);
+        assert.equal(editor.session.lineWidgets[0].el.innerHTML, `<div><span class="ace_ghost_text">Ghost2</span></div><div><span class="ace_ghost_text">Ghost3</span><span></span></div>`);
 
         editor.removeGhostText();
 
@@ -395,7 +395,7 @@ module.exports = {
         editor.renderer.$loop._flush();
         assert.equal(editor.renderer.content.textContent, "abcdefThis is a long test text that is longer than ");
 
-        assert.equal(editor.session.lineWidgets[0].el.innerHTML, `<div class="ghost_text_line_wrapped">30 characters</div><div> </div><div>Ghost3</div>`);
+        assert.equal(editor.session.lineWidgets[0].el.innerHTML, `<div class="ghost_text_line_wrapped"><span class="ace_ghost_text">30 characters</span></div><div><span class="ace_ghost_text"> </span></div><div><span class="ace_ghost_text">Ghost3</span><span></span></div>`);
 
         editor.removeGhostText();