rustdoc-search: replace TAB/NL/LF with SP first

This way, most of the parsing code doesn't need to be designed to handle it, since they should always be treated exactly the same anyhow.
rust-lang · Nov 29, 2023 · 930cba8 · 930cba8
1 parent 93f1711
commit 930cba8
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 23 deletions.
diff --git a/src/librustdoc/html/static/js/search.js b/src/librustdoc/html/static/js/search.js
@@ -287,10 +287,6 @@ function initSearch(rawSearchIndex) {
         }
     }
 
-    function isWhitespace(c) {
-        return " \t\n\r".indexOf(c) !== -1;
-    }
-
     function isSpecialStartCharacter(c) {
         return "<\"".indexOf(c) !== -1;
     }
@@ -408,7 +404,7 @@ function initSearch(rawSearchIndex) {
      * @return {boolean}
      */
     function isPathSeparator(c) {
-        return c === ":" || isWhitespace(c);
+        return c === ":" || c === " ";
     }
 
     /**
@@ -425,7 +421,7 @@ function initSearch(rawSearchIndex) {
             const c = parserState.userQuery[pos - 1];
             if (c === lookingFor) {
                 return true;
-            } else if (!isWhitespace(c)) {
+            } else if (c !== " ") {
                 break;
             }
             pos -= 1;
@@ -454,7 +450,7 @@ function initSearch(rawSearchIndex) {
     function skipWhitespace(parserState) {
         while (parserState.pos < parserState.userQuery.length) {
             const c = parserState.userQuery[parserState.pos];
-            if (!isWhitespace(c)) {
+            if (c !== " ") {
                 break;
             }
             parserState.pos += 1;
@@ -599,7 +595,7 @@ function initSearch(rawSearchIndex) {
                     } else {
                         while (parserState.pos + 1 < parserState.length) {
                             const next_c = parserState.userQuery[parserState.pos + 1];
-                            if (!isWhitespace(next_c)) {
+                            if (next_c !== " ") {
                                 break;
                             }
                             parserState.pos += 1;
@@ -953,7 +949,7 @@ function initSearch(rawSearchIndex) {
                 query.literalSearch = false;
                 foundStopChar = true;
                 continue;
-            } else if (isWhitespace(c)) {
+            } else if (c === " ") {
                 skipWhitespace(parserState);
                 continue;
             }
@@ -1113,7 +1109,7 @@ function initSearch(rawSearchIndex) {
                 }
             }
         }
-        userQuery = userQuery.trim();
+        userQuery = userQuery.trim().replace(/\r|\n|\t/g, " ");
         const parserState = {
             length: userQuery.length,
             pos: 0,

diff --git a/tests/rustdoc-js-std/parser-errors.js b/tests/rustdoc-js-std/parser-errors.js
@@ -161,6 +161,15 @@ const PARSED = [
         userQuery: "a:: ::b",
         error: "Unexpected `:: ::`",
     },
+    {
+        query: "a::\t::b",
+        elems: [],
+        foundElems: 0,
+        original: "a:: ::b",
+        returned: [],
+        userQuery: "a:: ::b",
+        error: "Unexpected `:: ::`",
+    },
     {
         query: "a::b::",
         elems: [],

diff --git a/tests/rustdoc-js-std/parser-separators.js b/tests/rustdoc-js-std/parser-separators.js
@@ -5,7 +5,7 @@ const PARSED = [
         query: 'aaaaaa	b',
         elems: [
             {
-                name: 'aaaaaa\tb',
+                name: 'aaaaaa b',
                 fullPath: ['aaaaaa', 'b'],
                 pathWithoutLast: ['aaaaaa'],
                 pathLast: 'b',
@@ -14,9 +14,9 @@ const PARSED = [
             },
         ],
         foundElems: 1,
-        original: "aaaaaa	b",
+        original: "aaaaaa b",
         returned: [],
-        userQuery: "aaaaaa	b",
+        userQuery: "aaaaaa b",
         error: null,
     },
     {
@@ -40,9 +40,9 @@ const PARSED = [
             },
         ],
         foundElems: 2,
-        original: "aaaaaa,	b",
+        original: "aaaaaa, b",
         returned: [],
-        userQuery: "aaaaaa,	b",
+        userQuery: "aaaaaa, b",
         error: null,
     },
     {
@@ -93,7 +93,7 @@ const PARSED = [
         query: 'a\tb',
         elems: [
             {
-                name: 'a\tb',
+                name: 'a b',
                 fullPath: ['a', 'b'],
                 pathWithoutLast: ['a'],
                 pathLast: 'b',
@@ -102,9 +102,9 @@ const PARSED = [
             },
         ],
         foundElems: 1,
-        original: "a\tb",
+        original: "a b",
         returned: [],
-        userQuery: "a\tb",
+        userQuery: "a b",
         error: null,
     },
     {
@@ -176,7 +176,7 @@ const PARSED = [
                 pathLast: 'a',
                 generics: [
                     {
-                        name: 'b\tc',
+                        name: 'b c',
                         fullPath: ['b', 'c'],
                         pathWithoutLast: ['b'],
                         pathLast: 'c',
@@ -187,9 +187,9 @@ const PARSED = [
             },
         ],
         foundElems: 1,
-        original: "a<b\tc>",
+        original: "a<b c>",
         returned: [],
-        userQuery: "a<b\tc>",
+        userQuery: "a<b c>",
         error: null,
     },
 ];
diff --git a/tests/rustdoc-js-std/parser-weird-queries.js b/tests/rustdoc-js-std/parser-weird-queries.js
@@ -92,9 +92,9 @@ const PARSED = [
         query: 'mod\t:',
         elems: [],
         foundElems: 0,
-        original: 'mod\t:',
+        original: 'mod :',
         returned: [],
-        userQuery: 'mod\t:',
+        userQuery: 'mod :',
         error: "Unexpected `:` (expected path after type filter `mod:`)",
     },
 ];