From 5b44bfda7fc62b2874400e613672aefe5b49aaaa Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Sat, 16 Mar 2024 17:50:44 -0700 Subject: [PATCH] rustdoc-search: shard the search result descriptions The descriptions are, on almost all crates[^1], the majority of the size of the search index, even though they aren't really used for searching. This makes it relatively easy to separate them into their own files. This commit also bumps us to ES8. Out of the browsers we support, all of them support async functions according to caniuse. https://caniuse.com/async-functions [^1]: , a crate with 44MiB of pure names and no descriptions for them, is an outlier and should not be counted. --- .../docker/host-x86_64/mingw-check/Dockerfile | 2 +- src/librustdoc/html/render/mod.rs | 33 +- src/librustdoc/html/render/search_index.rs | 101 ++++- src/librustdoc/html/render/write_shared.rs | 31 +- src/librustdoc/html/static/.eslintrc.js | 2 +- src/librustdoc/html/static/js/main.js | 28 +- src/librustdoc/html/static/js/search.js | 348 ++++++++++-------- src/librustdoc/html/static/js/storage.js | 4 +- src/tools/rustdoc-js/.eslintrc.js | 2 +- src/tools/rustdoc-js/tester.js | 104 ++++-- tests/rustdoc/search-index-summaries.rs | 2 +- 11 files changed, 428 insertions(+), 229 deletions(-) diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 30d3a52d82b82..de8db8ee03496 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -56,7 +56,7 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \ /scripts/validate-error-codes.sh && \ reuse --include-submodules lint && \ # Runs checks to ensure that there are no ES5 issues in our JS code. - es-check es6 ../src/librustdoc/html/static/js/*.js && \ + es-check es8 ../src/librustdoc/html/static/js/*.js && \ eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \ eslint -c ../src/tools/rustdoc-js/.eslintrc.js ../src/tools/rustdoc-js/tester.js && \ eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs index 6c5040414bced..c1a7593c26f08 100644 --- a/src/librustdoc/html/render/mod.rs +++ b/src/librustdoc/html/render/mod.rs @@ -184,40 +184,15 @@ pub(crate) enum RenderTypeId { impl RenderTypeId { pub fn write_to_string(&self, string: &mut String) { - // (sign, value) - let (sign, id): (bool, u32) = match &self { + let id: i32 = match &self { // 0 is a sentinel, everything else is one-indexed // concrete type - RenderTypeId::Index(idx) if *idx >= 0 => (false, (idx + 1isize).try_into().unwrap()), + RenderTypeId::Index(idx) if *idx >= 0 => (idx + 1isize).try_into().unwrap(), // generic type parameter - RenderTypeId::Index(idx) => (true, (-*idx).try_into().unwrap()), + RenderTypeId::Index(idx) => (*idx).try_into().unwrap(), _ => panic!("must convert render types to indexes before serializing"), }; - // zig-zag encoding - let value: u32 = (id << 1) | (if sign { 1 } else { 0 }); - // Self-terminating hex use capital letters for everything but the - // least significant digit, which is lowercase. For example, decimal 17 - // would be `` Aa `` if zig-zag encoding weren't used. - // - // Zig-zag encoding, however, stores the sign bit as the last bit. - // This means, in the last hexit, 1 is actually `c`, -1 is `b` - // (`a` is the imaginary -0), and, because all the bits are shifted - // by one, `` A` `` is actually 8 and `` Aa `` is -8. - // - // https://rust-lang.github.io/rustc-dev-guide/rustdoc-internals/search.html - // describes the encoding in more detail. - let mut shift: u32 = 28; - let mut mask: u32 = 0xF0_00_00_00; - while shift < 32 { - let hexit = (value & mask) >> shift; - if hexit != 0 || shift == 0 { - let hex = - char::try_from(if shift == 0 { '`' } else { '@' } as u32 + hexit).unwrap(); - string.push(hex); - } - shift = shift.wrapping_sub(4); - mask = mask >> 4; - } + search_index::write_vlqhex_to_string(id, string); } } diff --git a/src/librustdoc/html/render/search_index.rs b/src/librustdoc/html/render/search_index.rs index f153a90832910..34a4a89aa7bfa 100644 --- a/src/librustdoc/html/render/search_index.rs +++ b/src/librustdoc/html/render/search_index.rs @@ -17,12 +17,25 @@ use crate::html::format::join_with_double_colon; use crate::html::markdown::short_markdown_summary; use crate::html::render::{self, IndexItem, IndexItemFunctionType, RenderType, RenderTypeId}; +/// The serialized search description sharded version +/// +/// The `index` is a JSON-encoded list of names and other information. +/// +/// The desc has newlined descriptions, split up by size into 1MiB shards. +/// For example, `(4, "foo\nbar\nbaz\nquux")`. +pub(crate) struct SerializedSearchIndex { + pub(crate) index: String, + pub(crate) desc: Vec<(usize, String)>, +} + +const DESC_INDEX_SHARD_LEN: usize = 1024 * 1024; + /// Builds the search index from the collected metadata pub(crate) fn build_index<'tcx>( krate: &clean::Crate, cache: &mut Cache, tcx: TyCtxt<'tcx>, -) -> String { +) -> SerializedSearchIndex { let mut itemid_to_pathid = FxHashMap::default(); let mut primitives = FxHashMap::default(); let mut associated_types = FxHashMap::default(); @@ -318,7 +331,6 @@ pub(crate) fn build_index<'tcx>( .collect::>(); struct CrateData<'a> { - doc: String, items: Vec<&'a IndexItem>, paths: Vec<(ItemType, Vec)>, // The String is alias name and the vec is the list of the elements with this alias. @@ -327,6 +339,9 @@ pub(crate) fn build_index<'tcx>( aliases: &'a BTreeMap>, // Used when a type has more than one impl with an associated item with the same name. associated_item_disambiguators: &'a Vec<(usize, String)>, + // A list of shard lengths encoded as vlqhex. See the comment in write_vlqhex_to_string + // for information on the format. + descindex: String, } struct Paths { @@ -408,7 +423,6 @@ pub(crate) fn build_index<'tcx>( let mut names = Vec::with_capacity(self.items.len()); let mut types = String::with_capacity(self.items.len()); let mut full_paths = Vec::with_capacity(self.items.len()); - let mut descriptions = Vec::with_capacity(self.items.len()); let mut parents = Vec::with_capacity(self.items.len()); let mut functions = String::with_capacity(self.items.len()); let mut deprecated = Vec::with_capacity(self.items.len()); @@ -431,7 +445,6 @@ pub(crate) fn build_index<'tcx>( parents.push(item.parent_idx.map(|x| x + 1).unwrap_or(0)); names.push(item.name.as_str()); - descriptions.push(&item.desc); if !item.path.is_empty() { full_paths.push((index, &item.path)); @@ -454,14 +467,12 @@ pub(crate) fn build_index<'tcx>( let has_aliases = !self.aliases.is_empty(); let mut crate_data = serializer.serialize_struct("CrateData", if has_aliases { 9 } else { 8 })?; - crate_data.serialize_field("doc", &self.doc)?; crate_data.serialize_field("t", &types)?; crate_data.serialize_field("n", &names)?; - // Serialize as an array of item indices and full paths crate_data.serialize_field("q", &full_paths)?; - crate_data.serialize_field("d", &descriptions)?; crate_data.serialize_field("i", &parents)?; crate_data.serialize_field("f", &functions)?; + crate_data.serialize_field("D", &self.descindex)?; crate_data.serialize_field("c", &deprecated)?; crate_data.serialize_field("p", &paths)?; crate_data.serialize_field("b", &self.associated_item_disambiguators)?; @@ -472,16 +483,46 @@ pub(crate) fn build_index<'tcx>( } } - // Collect the index into a string - format!( + let desc = { + let mut result = Vec::new(); + let mut set = String::new(); + let mut len: usize = 0; + for desc in std::iter::once(&crate_doc).chain(crate_items.iter().map(|item| &item.desc)) { + if set.len() >= DESC_INDEX_SHARD_LEN { + result.push((len, std::mem::replace(&mut set, String::new()))); + len = 0; + } else if len != 0 { + set.push('\n'); + } + set.push_str(&desc); + len += 1; + } + result.push((len, std::mem::replace(&mut set, String::new()))); + result + }; + + let descindex = { + let mut descindex = String::with_capacity(desc.len() * 4); + for &(len, _) in desc.iter() { + write_vlqhex_to_string(len.try_into().unwrap(), &mut descindex); + } + descindex + }; + + assert_eq!(crate_items.len() + 1, desc.iter().map(|(len, _)| *len).sum::()); + + // The index, which is actually used to search, is JSON + // It uses `JSON.parse(..)` to actually load, since JSON + // parses faster than the full JavaScript syntax. + let index = format!( r#"["{}",{}]"#, krate.name(tcx), serde_json::to_string(&CrateData { - doc: crate_doc, items: crate_items, paths: crate_paths, aliases: &aliases, associated_item_disambiguators: &associated_item_disambiguators, + descindex, }) .expect("failed serde conversion") // All these `replace` calls are because we have to go through JS string for JSON content. @@ -489,7 +530,45 @@ pub(crate) fn build_index<'tcx>( .replace('\'', r"\'") // We need to escape double quotes for the JSON. .replace("\\\"", "\\\\\"") - ) + ); + SerializedSearchIndex { index, desc } +} + +pub(crate) fn write_vlqhex_to_string(n: i32, string: &mut String) { + let (sign, magnitude): (bool, u32) = + if n >= 0 { (false, n.try_into().unwrap()) } else { (true, (-n).try_into().unwrap()) }; + // zig-zag encoding + let value: u32 = (magnitude << 1) | (if sign { 1 } else { 0 }); + // Self-terminating hex use capital letters for everything but the + // least significant digit, which is lowercase. For example, decimal 17 + // would be `` Aa `` if zig-zag encoding weren't used. + // + // Zig-zag encoding, however, stores the sign bit as the last bit. + // This means, in the last hexit, 1 is actually `c`, -1 is `b` + // (`a` is the imaginary -0), and, because all the bits are shifted + // by one, `` A` `` is actually 8 and `` Aa `` is -8. + // + // https://rust-lang.github.io/rustc-dev-guide/rustdoc-internals/search.html + // describes the encoding in more detail. + let mut shift: u32 = 28; + let mut mask: u32 = 0xF0_00_00_00; + // first skip leading zeroes + while shift < 32 { + let hexit = (value & mask) >> shift; + if hexit != 0 || shift == 0 { + break; + } + shift = shift.wrapping_sub(4); + mask = mask >> 4; + } + // now write the rest + while shift < 32 { + let hexit = (value & mask) >> shift; + let hex = char::try_from(if shift == 0 { '`' } else { '@' } as u32 + hexit).unwrap(); + string.push(hex); + shift = shift.wrapping_sub(4); + mask = mask >> 4; + } } pub(crate) fn get_function_type_for_search<'tcx>( diff --git a/src/librustdoc/html/render/write_shared.rs b/src/librustdoc/html/render/write_shared.rs index fbd45b2b48ef9..c806bf1cc66f3 100644 --- a/src/librustdoc/html/render/write_shared.rs +++ b/src/librustdoc/html/render/write_shared.rs @@ -24,6 +24,7 @@ use crate::formats::cache::Cache; use crate::formats::item_type::ItemType; use crate::formats::Impl; use crate::html::format::Buffer; +use crate::html::render::search_index::SerializedSearchIndex; use crate::html::render::{AssocItemLink, ImplRenderingParameters}; use crate::html::{layout, static_files}; use crate::visit::DocVisitor; @@ -46,7 +47,7 @@ use crate::{try_err, try_none}; pub(super) fn write_shared( cx: &mut Context<'_>, krate: &Crate, - search_index: String, + search_index: SerializedSearchIndex, options: &RenderOptions, ) -> Result<(), Error> { // Write out the shared files. Note that these are shared among all rustdoc @@ -312,7 +313,7 @@ pub(super) fn write_shared( let dst = cx.dst.join(&format!("search-index{}.js", cx.shared.resource_suffix)); let (mut all_indexes, mut krates) = try_err!(collect_json(&dst, krate.name(cx.tcx()).as_str()), &dst); - all_indexes.push(search_index); + all_indexes.push(search_index.index); krates.push(krate.name(cx.tcx()).to_string()); krates.sort(); @@ -335,6 +336,32 @@ else if (window.initSearch) window.initSearch(searchIndex); Ok(v.into_bytes()) })?; + let search_desc_dir = cx.dst.join(format!("search.desc/{krate}", krate = krate.name(cx.tcx()))); + if Path::new(&search_desc_dir).exists() { + try_err!(std::fs::remove_dir_all(&search_desc_dir), &search_desc_dir); + } + try_err!(std::fs::create_dir_all(&search_desc_dir), &search_desc_dir); + let kratename = krate.name(cx.tcx()).to_string(); + for (i, (_, data)) in search_index.desc.into_iter().enumerate() { + let output_filename = static_files::suffix_path( + &format!("{kratename}-desc-{i}-.js"), + &cx.shared.resource_suffix, + ); + let path = search_desc_dir.join(output_filename); + try_err!( + std::fs::write( + &path, + &format!( + r##"searchState.loadedDescShard({kratename}, {i}, {data})"##, + kratename = serde_json::to_string(&kratename).unwrap(), + data = serde_json::to_string(&data).unwrap(), + ) + .into_bytes() + ), + &path + ); + } + write_invocation_specific("crates.js", &|| { let krates = krates.iter().map(|k| format!("\"{k}\"")).join(","); Ok(format!("window.ALL_CRATES = [{krates}];").into_bytes()) diff --git a/src/librustdoc/html/static/.eslintrc.js b/src/librustdoc/html/static/.eslintrc.js index 1a34530c2d16e..a1e9cc6dfa142 100644 --- a/src/librustdoc/html/static/.eslintrc.js +++ b/src/librustdoc/html/static/.eslintrc.js @@ -5,7 +5,7 @@ module.exports = { }, "extends": "eslint:recommended", "parserOptions": { - "ecmaVersion": 2015, + "ecmaVersion": 8, "sourceType": "module" }, "rules": { diff --git a/src/librustdoc/html/static/js/main.js b/src/librustdoc/html/static/js/main.js index b9a769a7c6da4..2c1330614b8d8 100644 --- a/src/librustdoc/html/static/js/main.js +++ b/src/librustdoc/html/static/js/main.js @@ -329,6 +329,26 @@ function preLoadCss(cssUrl) { search.innerHTML = "

" + searchState.loadingText + "

"; searchState.showResults(search); }, + descShards: new Map(), + loadDesc: async function({descShard, descIndex}) { + if (descShard.promise === null) { + descShard.promise = new Promise((resolve, reject) => { + descShard.resolve = resolve; + const ds = descShard; + const fname = `${ds.crate}-desc-${ds.shard}-`; + const url = resourcePath( + `search.desc/${descShard.crate}/${fname}`, + ".js", + ); + loadScript(url, reject); + }); + } + const list = await descShard.promise; + return list[descIndex]; + }, + loadedDescShard: function (crate, shard, data) { + this.descShards.get(crate)[shard].resolve(data.split("\n")); + }, }; const toggleAllDocsId = "toggle-all-docs"; @@ -381,7 +401,7 @@ function preLoadCss(cssUrl) { window.location.replace("#" + item.id); }, 0); } - } + }, ); } } @@ -585,7 +605,7 @@ function preLoadCss(cssUrl) { const script = document .querySelector("script[data-ignore-extern-crates]"); const ignoreExternCrates = new Set( - (script ? script.getAttribute("data-ignore-extern-crates") : "").split(",") + (script ? script.getAttribute("data-ignore-extern-crates") : "").split(","), ); for (const lib of libs) { if (lib === window.currentCrate || ignoreExternCrates.has(lib)) { @@ -1098,7 +1118,7 @@ function preLoadCss(cssUrl) { } else { wrapper.style.setProperty( "--popover-arrow-offset", - (wrapperPos.right - pos.right + 4) + "px" + (wrapperPos.right - pos.right + 4) + "px", ); } wrapper.style.visibility = ""; @@ -1680,7 +1700,7 @@ href="https://doc.rust-lang.org/${channel}/rustdoc/read-documentation/search.htm pendingSidebarResizingFrame = false; document.documentElement.style.setProperty( "--resizing-sidebar-width", - desiredSidebarSize + "px" + desiredSidebarSize + "px", ); }, 100); } diff --git a/src/librustdoc/html/static/js/search.js b/src/librustdoc/html/static/js/search.js index cbfa503f26060..2732d6b15436d 100644 --- a/src/librustdoc/html/static/js/search.js +++ b/src/librustdoc/html/static/js/search.js @@ -206,14 +206,14 @@ const editDistanceState = { // insertion this.current[j - 1] + 1, // substitution - this.prev[j - 1] + substitutionCost + this.prev[j - 1] + substitutionCost, ); if ((i > 1) && (j > 1) && (a[aIdx] === b[bIdx - 1]) && (a[aIdx - 1] === b[bIdx])) { // transposition this.current[j] = Math.min( this.current[j], - this.prevPrev[j - 2] + 1 + this.prevPrev[j - 2] + 1, ); } } @@ -856,8 +856,8 @@ function initSearch(rawSearchIndex) { parserState, parserState.userQuery.slice(start, end), generics, - isInGenerics - ) + isInGenerics, + ), ); } } @@ -1295,7 +1295,7 @@ function initSearch(rawSearchIndex) { * * @return {ResultsTable} */ - function execQuery(parsedQuery, filterCrates, currentCrate) { + async function execQuery(parsedQuery, filterCrates, currentCrate) { const results_others = new Map(), results_in_args = new Map(), results_returned = new Map(); @@ -1326,6 +1326,7 @@ function initSearch(rawSearchIndex) { duplicates.add(obj.fullPath); obj.href = res[1]; + obj.desc = result.desc; out.push(obj); if (out.length >= MAX_RESULTS) { break; @@ -1342,9 +1343,9 @@ function initSearch(rawSearchIndex) { * @param {Results} results * @param {boolean} isType * @param {string} preferredCrate - * @returns {[ResultObject]} + * @returns {Promise<[ResultObject]>} */ - function sortResults(results, isType, preferredCrate) { + async function sortResults(results, isType, preferredCrate) { const userQuery = parsedQuery.userQuery; const result_list = []; for (const result of results.values()) { @@ -1352,6 +1353,12 @@ function initSearch(rawSearchIndex) { result.word = searchIndex[result.id].word; result_list.push(result); } + for (const result of result_list) { + result.desc = searchState.loadDesc(result.item); + } + for (const result of result_list) { + result.desc = await result.desc; + } result_list.sort((aaa, bbb) => { let a, b; @@ -1422,8 +1429,8 @@ function initSearch(rawSearchIndex) { } // sort by description (no description goes later) - a = (aaa.item.desc === ""); - b = (bbb.item.desc === ""); + a = (aaa.desc === ""); + b = (bbb.desc === ""); if (a !== b) { return a - b; } @@ -1477,7 +1484,7 @@ function initSearch(rawSearchIndex) { whereClause, mgensIn, solutionCb, - unboxingDepth + unboxingDepth, ) { if (unboxingDepth >= UNBOXING_LIMIT) { return false; @@ -1524,7 +1531,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + 1 + unboxingDepth + 1, )) { continue; } @@ -1541,7 +1548,7 @@ function initSearch(rawSearchIndex) { whereClause, mgensScratch, solutionCb, - unboxingDepth + 1 + unboxingDepth + 1, )) { return true; } @@ -1551,7 +1558,7 @@ function initSearch(rawSearchIndex) { whereClause, mgens ? new Map(mgens) : null, solutionCb, - unboxingDepth + 1 + unboxingDepth + 1, )) { return true; } @@ -1625,7 +1632,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgensScratch, - unboxingDepth + unboxingDepth, ); if (!solution) { return false; @@ -1638,7 +1645,7 @@ function initSearch(rawSearchIndex) { whereClause, simplifiedMgens, solutionCb, - unboxingDepth + unboxingDepth, ); if (passesUnification) { return true; @@ -1646,7 +1653,7 @@ function initSearch(rawSearchIndex) { } return false; }, - unboxingDepth + unboxingDepth, ); if (passesUnification) { return true; @@ -1663,7 +1670,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + 1 + unboxingDepth + 1, )) { continue; } @@ -1689,7 +1696,7 @@ function initSearch(rawSearchIndex) { whereClause, mgensScratch, solutionCb, - unboxingDepth + 1 + unboxingDepth + 1, ); if (passesUnification) { return true; @@ -1820,7 +1827,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgensIn, - unboxingDepth + unboxingDepth, ) { if (fnType.bindings.size < queryElem.bindings.size) { return false; @@ -1849,7 +1856,7 @@ function initSearch(rawSearchIndex) { // possible solutions return false; }, - unboxingDepth + unboxingDepth, ); return newSolutions; }); @@ -1887,7 +1894,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + unboxingDepth, ) { if (unboxingDepth >= UNBOXING_LIMIT) { return false; @@ -1914,7 +1921,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgensTmp, - unboxingDepth + unboxingDepth, ); } else if (fnType.generics.length > 0 || fnType.bindings.size > 0) { const simplifiedGenerics = [ @@ -1926,7 +1933,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + unboxingDepth, ); } return false; @@ -1975,7 +1982,7 @@ function initSearch(rawSearchIndex) { elem, whereClause, mgens, - unboxingDepth + 1 + unboxingDepth + 1, ); } if (row.id > 0 && elem.id > 0 && elem.pathWithoutLast.length === 0 && @@ -1989,7 +1996,7 @@ function initSearch(rawSearchIndex) { elem, whereClause, mgens, - unboxingDepth + unboxingDepth, ); } } @@ -2007,7 +2014,7 @@ function initSearch(rawSearchIndex) { return 0; } const maxPathEditDistance = Math.floor( - contains.reduce((acc, next) => acc + next.length, 0) / 3 + contains.reduce((acc, next) => acc + next.length, 0) / 3, ); let ret_dist = maxPathEditDistance + 1; const path = ty.path.split("::"); @@ -2066,7 +2073,8 @@ function initSearch(rawSearchIndex) { crate: item.crate, name: item.name, path: item.path, - desc: item.desc, + descShard: item.descShard, + descIndex: item.descIndex, ty: item.ty, parent: item.parent, type: item.type, @@ -2192,7 +2200,7 @@ function initSearch(rawSearchIndex) { results_others, results_in_args, results_returned, - maxEditDistance + maxEditDistance, ) { if (!row || (filterCrates !== null && row.crate !== filterCrates)) { return; @@ -2204,7 +2212,7 @@ function initSearch(rawSearchIndex) { // atoms in the function not present in the query const tfpDist = compareTypeFingerprints( fullId, - parsedQuery.typeFingerprint + parsedQuery.typeFingerprint, ); if (tfpDist !== null) { const in_args = row.type && row.type.inputs @@ -2276,7 +2284,7 @@ function initSearch(rawSearchIndex) { const tfpDist = compareTypeFingerprints( row.id, - parsedQuery.typeFingerprint + parsedQuery.typeFingerprint, ); if (tfpDist === null) { return; @@ -2298,10 +2306,10 @@ function initSearch(rawSearchIndex) { row.type.where_clause, mgens, null, - 0 // unboxing depth + 0, // unboxing depth ); }, - 0 // unboxing depth + 0, // unboxing depth )) { return; } @@ -2419,7 +2427,7 @@ function initSearch(rawSearchIndex) { } return [typeNameIdMap.get(name).id, constraints]; - }) + }), ); } @@ -2446,7 +2454,7 @@ function initSearch(rawSearchIndex) { results_others, results_in_args, results_returned, - maxEditDistance + maxEditDistance, ); } } @@ -2478,9 +2486,9 @@ function initSearch(rawSearchIndex) { } const ret = createQueryResults( - sortResults(results_in_args, true, currentCrate), - sortResults(results_returned, true, currentCrate), - sortResults(results_others, false, currentCrate), + await sortResults(results_in_args, true, currentCrate), + await sortResults(results_returned, true, currentCrate), + await sortResults(results_others, false, currentCrate), parsedQuery); handleAliases(ret, parsedQuery.original.replace(/"/g, ""), filterCrates, currentCrate); if (parsedQuery.error !== null && ret.others.length !== 0) { @@ -2581,14 +2589,14 @@ function initSearch(rawSearchIndex) { * @param {ParsedQuery} query * @param {boolean} display - True if this is the active tab */ - function addTab(array, query, display) { + async function addTab(array, query, display) { const extraClass = display ? " active" : ""; const output = document.createElement("div"); if (array.length > 0) { output.className = "search-results " + extraClass; - array.forEach(item => { + for (const item of array) { const name = item.name; const type = itemTypes[item.ty]; const longType = longItemTypes[item.ty]; @@ -2624,7 +2632,7 @@ ${item.displayPath}${name}\ link.appendChild(description); output.appendChild(link); - }); + } } else if (query.error === null) { output.className = "search-failed" + extraClass; output.innerHTML = "No results :(
" + @@ -2666,7 +2674,7 @@ ${item.displayPath}${name}\ * @param {boolean} go_to_first * @param {string} filterCrates */ - function showResults(results, go_to_first, filterCrates) { + async function showResults(results, go_to_first, filterCrates) { const search = searchState.outputElement(); if (go_to_first || (results.others.length === 1 && getSettingValue("go-to-only-result") === "true") @@ -2699,9 +2707,9 @@ ${item.displayPath}${name}\ currentResults = results.query.userQuery; - const ret_others = addTab(results.others, results.query, true); - const ret_in_args = addTab(results.in_args, results.query, false); - const ret_returned = addTab(results.returned, results.query, false); + const ret_others = await addTab(results.others, results.query, true); + const ret_in_args = await addTab(results.in_args, results.query, false); + const ret_returned = await addTab(results.returned, results.query, false); // Navigate to the relevant tab if the current tab is empty, like in case users search // for "-> String". If they had selected another tab previously, they have to click on @@ -2822,7 +2830,7 @@ ${item.displayPath}${name}\ * and display the results. * @param {boolean} [forced] */ - function search(forced) { + async function search(forced) { const query = parseQuery(searchState.input.value.trim()); let filterCrates = getFilterCrates(); @@ -2850,8 +2858,8 @@ ${item.displayPath}${name}\ // recent search query is added to the browser history. updateSearchHistory(buildUrl(query.original, filterCrates)); - showResults( - execQuery(query, filterCrates, window.currentCrate), + await showResults( + await execQuery(query, filterCrates, window.currentCrate), params.go_to_first, filterCrates); } @@ -2920,7 +2928,7 @@ ${item.displayPath}${name}\ pathIndex = type[PATH_INDEX_DATA]; generics = buildItemSearchTypeAll( type[GENERICS_DATA], - lowercasePaths + lowercasePaths, ); if (type.length > BINDINGS_DATA && type[BINDINGS_DATA].length > 0) { bindings = new Map(type[BINDINGS_DATA].map(binding => { @@ -3030,101 +3038,49 @@ ${item.displayPath}${name}\ * The raw function search type format is generated using serde in * librustdoc/html/render/mod.rs: IndexItemFunctionType::write_to_string * - * @param {{ - * string: string, - * offset: number, - * backrefQueue: FunctionSearchType[] - * }} itemFunctionDecoder * @param {Array<{name: string, ty: number}>} lowercasePaths - * @param {Map} * * @return {null|FunctionSearchType} */ - function buildFunctionSearchType(itemFunctionDecoder, lowercasePaths) { - const c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - itemFunctionDecoder.offset += 1; - const [zero, ua, la, ob, cb] = ["0", "@", "`", "{", "}"].map(c => c.charCodeAt(0)); - // `` ` `` is used as a sentinel because it's fewer bytes than `null`, and decodes to zero - // `0` is a backref - if (c === la) { - return null; - } - // sixteen characters after "0" are backref - if (c >= zero && c < ua) { - return itemFunctionDecoder.backrefQueue[c - zero]; - } - if (c !== ob) { - throw ["Unexpected ", c, " in function: expected ", "{", "; this is a bug"]; - } - // call after consuming `{` - function decodeList() { - let c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - const ret = []; - while (c !== cb) { - ret.push(decode()); - c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - } - itemFunctionDecoder.offset += 1; // eat cb - return ret; - } - // consumes and returns a list or integer - function decode() { - let n = 0; - let c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - if (c === ob) { - itemFunctionDecoder.offset += 1; - return decodeList(); - } - while (c < la) { - n = (n << 4) | (c & 0xF); - itemFunctionDecoder.offset += 1; - c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - } - // last character >= la - n = (n << 4) | (c & 0xF); - const [sign, value] = [n & 1, n >> 1]; - itemFunctionDecoder.offset += 1; - return sign ? -value : value; - } - const functionSearchType = decodeList(); - const INPUTS_DATA = 0; - const OUTPUT_DATA = 1; - let inputs, output; - if (typeof functionSearchType[INPUTS_DATA] === "number") { - inputs = [buildItemSearchType(functionSearchType[INPUTS_DATA], lowercasePaths)]; - } else { - inputs = buildItemSearchTypeAll( - functionSearchType[INPUTS_DATA], - lowercasePaths - ); - } - if (functionSearchType.length > 1) { - if (typeof functionSearchType[OUTPUT_DATA] === "number") { - output = [buildItemSearchType(functionSearchType[OUTPUT_DATA], lowercasePaths)]; + function buildFunctionSearchTypeCallback(lowercasePaths) { + return functionSearchType => { + if (functionSearchType === 0) { + return null; + } + const INPUTS_DATA = 0; + const OUTPUT_DATA = 1; + let inputs, output; + if (typeof functionSearchType[INPUTS_DATA] === "number") { + inputs = [buildItemSearchType(functionSearchType[INPUTS_DATA], lowercasePaths)]; } else { - output = buildItemSearchTypeAll( - functionSearchType[OUTPUT_DATA], - lowercasePaths + inputs = buildItemSearchTypeAll( + functionSearchType[INPUTS_DATA], + lowercasePaths, ); } - } else { - output = []; - } - const where_clause = []; - const l = functionSearchType.length; - for (let i = 2; i < l; ++i) { - where_clause.push(typeof functionSearchType[i] === "number" - ? [buildItemSearchType(functionSearchType[i], lowercasePaths)] - : buildItemSearchTypeAll(functionSearchType[i], lowercasePaths)); - } - const ret = { - inputs, output, where_clause, + if (functionSearchType.length > 1) { + if (typeof functionSearchType[OUTPUT_DATA] === "number") { + output = [buildItemSearchType(functionSearchType[OUTPUT_DATA], lowercasePaths)]; + } else { + output = buildItemSearchTypeAll( + functionSearchType[OUTPUT_DATA], + lowercasePaths, + ); + } + } else { + output = []; + } + const where_clause = []; + const l = functionSearchType.length; + for (let i = 2; i < l; ++i) { + where_clause.push(typeof functionSearchType[i] === "number" + ? [buildItemSearchType(functionSearchType[i], lowercasePaths)] + : buildItemSearchTypeAll(functionSearchType[i], lowercasePaths)); + } + return { + inputs, output, where_clause, + }; }; - itemFunctionDecoder.backrefQueue.unshift(ret); - if (itemFunctionDecoder.backrefQueue.length > 16) { - itemFunctionDecoder.backrefQueue.pop(); - } - return ret; } /** @@ -3245,6 +3201,68 @@ ${item.displayPath}${name}\ return functionTypeFingerprint[(fullId * 4) + 3]; } + class VlqHexDecoder { + constructor(string, cons) { + this.string = string; + this.cons = cons; + this.offset = 0; + this.backrefQueue = []; + } + // call after consuming `{` + decodeList() { + const cb = "}".charCodeAt(0); + let c = this.string.charCodeAt(this.offset); + const ret = []; + while (c !== cb) { + ret.push(this.decode()); + c = this.string.charCodeAt(this.offset); + } + this.offset += 1; // eat cb + return ret; + } + // consumes and returns a list or integer + decode() { + const [ob, la] = ["{", "`"].map(c => c.charCodeAt(0)); + let n = 0; + let c = this.string.charCodeAt(this.offset); + if (c === ob) { + this.offset += 1; + return this.decodeList(); + } + while (c < la) { + n = (n << 4) | (c & 0xF); + this.offset += 1; + c = this.string.charCodeAt(this.offset); + } + // last character >= la + n = (n << 4) | (c & 0xF); + const [sign, value] = [n & 1, n >> 1]; + this.offset += 1; + return sign ? -value : value; + } + next() { + const c = this.string.charCodeAt(this.offset); + const [zero, ua, la] = ["0", "@", "`"].map(c => c.charCodeAt(0)); + // sixteen characters after "0" are backref + if (c >= zero && c < ua) { + this.offset += 1; + return this.backrefQueue[c - zero]; + } + // special exception: 0 doesn't use backref encoding + // it's already one character, and it's always nullish + if (c === la) { + this.offset += 1; + return this.cons(0); + } + const result = this.cons(this.decode()); + this.backrefQueue.unshift(result); + if (this.backrefQueue.length > 16) { + this.backrefQueue.pop(); + } + return result; + } + } + /** * Convert raw search index into in-memory search index. * @@ -3271,18 +3289,32 @@ ${item.displayPath}${name}\ id = 0; for (const [crate, crateCorpus] of rawSearchIndex) { + // a string representing the lengths of each description shard + // a string representing the list of function types + const itemDescShardDecoder = new VlqHexDecoder(crateCorpus.D, noop => noop); + let descShard = { + crate, + shard: 0, + start: 0, + len: itemDescShardDecoder.next(), + promise: null, + resolve: null, + }; + const descShardList = [ descShard ]; + // This object should have exactly the same set of fields as the "row" // object defined below. Your JavaScript runtime will thank you. // https://mathiasbynens.be/notes/shapes-ics const crateRow = { - crate: crate, + crate, ty: 3, // == ExternCrate name: crate, path: "", - desc: crateCorpus.doc, + descShard, + descIndex: 0, parent: undefined, type: null, - id: id, + id, word: crate, normalizedName: crate.indexOf("_") === -1 ? crate : crate.replace(/_/g, ""), deprecated: null, @@ -3302,16 +3334,8 @@ ${item.displayPath}${name}\ // i.e. if indices 4 and 11 are present, but 5-10 and 12-13 are not present, // 5-10 will fall back to the path for 4 and 12-13 will fall back to the path for 11 const itemPaths = new Map(crateCorpus.q); - // an array of (String) descriptions - const itemDescs = crateCorpus.d; // an array of (Number) the parent path index + 1 to `paths`, or 0 if none const itemParentIdxs = crateCorpus.i; - // a string representing the list of function types - const itemFunctionDecoder = { - string: crateCorpus.f, - offset: 0, - backrefQueue: [], - }; // an array of (Number) indices for the deprecated items const deprecatedItems = new Set(crateCorpus.c); // an array of (Number) indices for the deprecated items @@ -3326,6 +3350,12 @@ ${item.displayPath}${name}\ // an array of [{name: String, ty: Number}] const lowercasePaths = []; + // a string representing the list of function types + const itemFunctionDecoder = new VlqHexDecoder( + crateCorpus.f, + buildFunctionSearchTypeCallback(lowercasePaths), + ); + // convert `rawPaths` entries into object form // generate normalizedPaths for function search mode let len = paths.length; @@ -3353,13 +3383,26 @@ ${item.displayPath}${name}\ // faster analysis operations lastPath = ""; len = itemTypes.length; + let descIndex = 1; for (let i = 0; i < len; ++i) { + if (descIndex >= descShard.len) { + descShard = { + crate, + shard: descShard.shard + 1, + start: descShard.start + descShard.len, + len: itemDescShardDecoder.next(), + promise: null, + resolve: null, + }; + descIndex = 0; + descShardList.push(descShard); + } let word = ""; if (typeof itemNames[i] === "string") { word = itemNames[i].toLowerCase(); } const path = itemPaths.has(i) ? itemPaths.get(i) : lastPath; - const type = buildFunctionSearchType(itemFunctionDecoder, lowercasePaths); + const type = itemFunctionDecoder.next(); if (type !== null) { if (type) { const fp = functionTypeFingerprint.subarray(id * 4, (id + 1) * 4); @@ -3380,14 +3423,15 @@ ${item.displayPath}${name}\ // This object should have exactly the same set of fields as the "crateRow" // object defined above. const row = { - crate: crate, + crate, ty: itemTypes.charCodeAt(i) - charA, name: itemNames[i], - path: path, - desc: itemDescs[i], + path, + descShard, + descIndex, parent: itemParentIdxs[i] > 0 ? paths[itemParentIdxs[i] - 1] : undefined, type, - id: id, + id, word, normalizedName: word.indexOf("_") === -1 ? word : word.replace(/_/g, ""), deprecated: deprecatedItems.has(i), @@ -3396,6 +3440,7 @@ ${item.displayPath}${name}\ id += 1; searchIndex.push(row); lastPath = row.path; + descIndex += 1; } if (aliases) { @@ -3419,6 +3464,7 @@ ${item.displayPath}${name}\ } } currentIndex += itemTypes.length; + searchState.descShards.set(crate, descShardList); } // Drop the (rather large) hash table used for reusing function items TYPES_POOL = new Map(); diff --git a/src/librustdoc/html/static/js/storage.js b/src/librustdoc/html/static/js/storage.js index bda7b3c647e7e..73c543567c078 100644 --- a/src/librustdoc/html/static/js/storage.js +++ b/src/librustdoc/html/static/js/storage.js @@ -211,14 +211,14 @@ function updateSidebarWidth() { if (desktopSidebarWidth && desktopSidebarWidth !== "null") { document.documentElement.style.setProperty( "--desktop-sidebar-width", - desktopSidebarWidth + "px" + desktopSidebarWidth + "px", ); } const srcSidebarWidth = getSettingValue("src-sidebar-width"); if (srcSidebarWidth && srcSidebarWidth !== "null") { document.documentElement.style.setProperty( "--src-sidebar-width", - srcSidebarWidth + "px" + srcSidebarWidth + "px", ); } } diff --git a/src/tools/rustdoc-js/.eslintrc.js b/src/tools/rustdoc-js/.eslintrc.js index 4ab3a31573309..b9d0e251c242b 100644 --- a/src/tools/rustdoc-js/.eslintrc.js +++ b/src/tools/rustdoc-js/.eslintrc.js @@ -6,7 +6,7 @@ module.exports = { }, "extends": "eslint:recommended", "parserOptions": { - "ecmaVersion": 2015, + "ecmaVersion": 8, "sourceType": "module" }, "rules": { diff --git a/src/tools/rustdoc-js/tester.js b/src/tools/rustdoc-js/tester.js index 86881ef362e9f..ae4121cc03f51 100644 --- a/src/tools/rustdoc-js/tester.js +++ b/src/tools/rustdoc-js/tester.js @@ -1,3 +1,4 @@ +/* global globalThis */ const fs = require("fs"); const path = require("path"); @@ -133,7 +134,7 @@ function valueCheck(fullPath, expected, result, error_text, queryName) { expected_value, result.get(key), error_text, - queryName + queryName, ); } else { error_text.push(`${queryName}==> EXPECTED has extra key in map from field ` + @@ -212,11 +213,11 @@ function runParser(query, expected, parseQuery, queryName) { return error_text; } -function runSearch(query, expected, doSearch, loadedFile, queryName) { +async function runSearch(query, expected, doSearch, loadedFile, queryName) { const ignore_order = loadedFile.ignore_order; const exact_check = loadedFile.exact_check; - const results = doSearch(query, loadedFile.FILTER_CRATE); + const results = await doSearch(query, loadedFile.FILTER_CRATE); const error_text = []; for (const key in expected) { @@ -238,7 +239,7 @@ function runSearch(query, expected, doSearch, loadedFile, queryName) { } let prev_pos = -1; - entry.forEach((elem, index) => { + for (const [index, elem] of entry.entries()) { const entry_pos = lookForEntry(elem, results[key]); if (entry_pos === -1) { error_text.push(queryName + "==> Result not found in '" + key + "': '" + @@ -260,13 +261,13 @@ function runSearch(query, expected, doSearch, loadedFile, queryName) { } else { prev_pos = entry_pos; } - }); + } } return error_text; } -function runCorrections(query, corrections, getCorrections, loadedFile) { - const qc = getCorrections(query, loadedFile.FILTER_CRATE); +async function runCorrections(query, corrections, getCorrections, loadedFile) { + const qc = await getCorrections(query, loadedFile.FILTER_CRATE); const error_text = []; if (corrections === null) { @@ -299,18 +300,27 @@ function checkResult(error_text, loadedFile, displaySuccess) { return 1; } -function runCheckInner(callback, loadedFile, entry, getCorrections, extra) { +async function runCheckInner(callback, loadedFile, entry, getCorrections, extra) { if (typeof entry.query !== "string") { console.log("FAILED"); console.log("==> Missing `query` field"); return false; } - let error_text = callback(entry.query, entry, extra ? "[ query `" + entry.query + "`]" : ""); + let error_text = await callback( + entry.query, + entry, + extra ? "[ query `" + entry.query + "`]" : "", + ); if (checkResult(error_text, loadedFile, false) !== 0) { return false; } if (entry.correction !== undefined) { - error_text = runCorrections(entry.query, entry.correction, getCorrections, loadedFile); + error_text = await runCorrections( + entry.query, + entry.correction, + getCorrections, + loadedFile, + ); if (checkResult(error_text, loadedFile, false) !== 0) { return false; } @@ -318,16 +328,16 @@ function runCheckInner(callback, loadedFile, entry, getCorrections, extra) { return true; } -function runCheck(loadedFile, key, getCorrections, callback) { +async function runCheck(loadedFile, key, getCorrections, callback) { const expected = loadedFile[key]; if (Array.isArray(expected)) { for (const entry of expected) { - if (!runCheckInner(callback, loadedFile, entry, getCorrections, true)) { + if (!await runCheckInner(callback, loadedFile, entry, getCorrections, true)) { return 1; } } - } else if (!runCheckInner(callback, loadedFile, expected, getCorrections, false)) { + } else if (!await runCheckInner(callback, loadedFile, expected, getCorrections, false)) { return 1; } console.log("OK"); @@ -338,7 +348,7 @@ function hasCheck(content, checkName) { return content.startsWith(`const ${checkName}`) || content.includes(`\nconst ${checkName}`); } -function runChecks(testFile, doSearch, parseQuery, getCorrections) { +async function runChecks(testFile, doSearch, parseQuery, getCorrections) { let checkExpected = false; let checkParsed = false; let testFileContent = readFile(testFile); @@ -367,12 +377,12 @@ function runChecks(testFile, doSearch, parseQuery, getCorrections) { let res = 0; if (checkExpected) { - res += runCheck(loadedFile, "EXPECTED", getCorrections, (query, expected, text) => { + res += await runCheck(loadedFile, "EXPECTED", getCorrections, (query, expected, text) => { return runSearch(query, expected, doSearch, loadedFile, text); }); } if (checkParsed) { - res += runCheck(loadedFile, "PARSED", getCorrections, (query, expected, text) => { + res += await runCheck(loadedFile, "PARSED", getCorrections, (query, expected, text) => { return runParser(query, expected, parseQuery, text); }); } @@ -393,6 +403,35 @@ function loadSearchJS(doc_folder, resource_suffix) { const searchIndexJs = path.join(doc_folder, "search-index" + resource_suffix + ".js"); const searchIndex = require(searchIndexJs); + globalThis.searchState = { + descShards: new Map(), + loadDesc: async function({descShard, descIndex}) { + if (descShard.promise === null) { + descShard.promise = new Promise((resolve, reject) => { + descShard.resolve = resolve; + const ds = descShard; + const fname = `${ds.crate}-desc-${ds.shard}-${resource_suffix}.js`; + fs.readFile( + `${doc_folder}/search.desc/${descShard.crate}/${fname}`, + (err, data) => { + if (err) { + reject(err); + } else { + eval(data.toString("utf8")); + } + }, + ); + }); + } + const list = await descShard.promise; + return list[descIndex]; + }, + loadedDescShard: function (crate, shard, data) { + //console.log(this.descShards); + this.descShards.get(crate)[shard].resolve(data.split("\n")); + }, + }; + const staticFiles = path.join(doc_folder, "static.files"); const searchJs = fs.readdirSync(staticFiles).find(f => f.match(/search.*\.js$/)); const searchModule = require(path.join(staticFiles, searchJs)); @@ -474,7 +513,7 @@ function parseOptions(args) { return null; } -function main(argv) { +async function main(argv) { const opts = parseOptions(argv.slice(2)); if (opts === null) { return 1; @@ -482,7 +521,7 @@ function main(argv) { const parseAndSearch = loadSearchJS( opts["doc_folder"], - opts["resource_suffix"] + opts["resource_suffix"], ); let errors = 0; @@ -494,21 +533,34 @@ function main(argv) { }; if (opts["test_file"].length !== 0) { - opts["test_file"].forEach(file => { + for (const file of opts["test_file"]) { process.stdout.write(`Testing ${file} ... `); - errors += runChecks(file, doSearch, parseAndSearch.parseQuery, getCorrections); - }); + errors += await runChecks(file, doSearch, parseAndSearch.parseQuery, getCorrections); + } } else if (opts["test_folder"].length !== 0) { - fs.readdirSync(opts["test_folder"]).forEach(file => { + for (const file of fs.readdirSync(opts["test_folder"])) { if (!file.endsWith(".js")) { - return; + continue; } process.stdout.write(`Testing ${file} ... `); - errors += runChecks(path.join(opts["test_folder"], file), doSearch, + errors += await runChecks(path.join(opts["test_folder"], file), doSearch, parseAndSearch.parseQuery, getCorrections); - }); + } } return errors > 0 ? 1 : 0; } -process.exit(main(process.argv)); +main(process.argv).catch(e => { + console.log(e); + process.exit(1); +}).then(x => process.exit(x)); + +process.on("beforeExit", () => { + console.log("process did not complete"); + process.exit(1); +}); + +/*process.on("uncaughtException", (err) => { + console.log(`Uncaught Exception: ${err.message}`); + process.exit(1); +});*/ diff --git a/tests/rustdoc/search-index-summaries.rs b/tests/rustdoc/search-index-summaries.rs index efd366405bfeb..529b42d0ca903 100644 --- a/tests/rustdoc/search-index-summaries.rs +++ b/tests/rustdoc/search-index-summaries.rs @@ -1,6 +1,6 @@ #![crate_name = "foo"] -// @hasraw 'search-index.js' 'Foo short link.' +// @hasraw 'search.desc/foo/foo-desc-0-.js' 'Foo short link.' // @!hasraw - 'www.example.com' // @!hasraw - 'More Foo.'