Explicitly yield while building index every 256 items

Author: notriddle

src/librustdoc/html/static/js/search.js

@@ -1450,11 +1450,10 @@ class NameTrie {
 
 class DocSearch {
     /**
-     * @param {Map<string, rustdoc.RawSearchIndexCrate>} rawSearchIndex
      * @param {string} rootPath
      * @param {rustdoc.SearchState} searchState
      */
-    constructor(rawSearchIndex, rootPath, searchState) {
+    constructor(rootPath, searchState) {
         /**
          * @type {Map<String, RoaringBitmap>}
          */
@@ -1585,7 +1584,7 @@ class DocSearch {
         /**
          *  @type {Array<rustdoc.Row>}
          */
-        this.searchIndex = this.buildIndex(rawSearchIndex);
+        this.searchIndex = [];
     }
 
     /**
@@ -1911,9 +1910,9 @@ class DocSearch {
      * Convert raw search index into in-memory search index.
      *
      * @param {Map<string, rustdoc.RawSearchIndexCrate>} rawSearchIndex
-     * @returns {rustdoc.Row[]}
+     * @returns {Promise<rustdoc.Row[]>}
      */
-    buildIndex(rawSearchIndex) {
+    async buildIndex(rawSearchIndex) {
         /**
          * Convert from RawFunctionSearchType to FunctionSearchType.
          *
@@ -2177,6 +2176,20 @@ class DocSearch {
                 paths[i] = { ty, name, path, exactPath, unboxFlag };
             }
 
+            // Throttlers are used to yield to the JavaScript event loop
+            // while this is being built.
+            // They're generated up-front to avoid the "nesting level"
+            // limit that limits our speed to 4ms per tick.
+            // https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html
+            const throttlers = [];
+            len = itemTypes.length;
+            for (let i = 0; i < len; ++i) {
+                if ((i & 0xFF) === 0) { // 256 - 1
+                    throttlers.push(new Promise(resolve => {
+                        setTimeout(resolve, 0);
+                    }));
+                }
+            }
             // convert `item*` into an object form, and construct word indices.
             //
             // before any analysis is performed lets gather the search terms to
@@ -2189,6 +2202,9 @@ class DocSearch {
             let lastName = "";
             let lastWord = "";
             for (let i = 0; i < len; ++i) {
+                if ((i & 0xFF) === 0) { // 256 - 1
+                    await throttlers[i >> 8];
+                }
                 const bitIndex = i + 1;
                 if (descIndex >= descShard.len &&
                     // @ts-expect-error
@@ -5377,20 +5393,26 @@ function updateCrate(ev) {
     search(true);
 }
 
-// @ts-expect-error
-function initSearch(searchIndx) {
+/**
+ * @param {Map<string, import("./rustdoc").RawSearchIndexCrate>} searchIndx 
+ */
+async function initSearch(searchIndx) {
+    if (ROOT_PATH === null) {
+        return;
+    }
     rawSearchIndex = searchIndx;
     if (typeof window !== "undefined") {
-        // @ts-expect-error
-        docSearch = new DocSearch(rawSearchIndex, ROOT_PATH, searchState);
+        docSearch = new DocSearch(ROOT_PATH, window.searchState);
+        docSearch.searchIndex = await docSearch.buildIndex(rawSearchIndex);
         registerSearchEvents();
         // If there's a search term in the URL, execute the search now.
         if (window.searchState.getQueryStringParams().search !== undefined) {
             search();
         }
     } else if (typeof exports !== "undefined") {
-        // @ts-expect-error
-        docSearch = new DocSearch(rawSearchIndex, ROOT_PATH, searchState);
+        // @ts-ignore
+        docSearch = new DocSearch(ROOT_PATH, searchState);
+        docSearch.searchIndex = await docSearch.buildIndex(rawSearchIndex);
         exports.docSearch = docSearch;
         exports.parseQuery = DocSearch.parseQuery;
     }

src/tools/rustdoc-js/tester.js

@@ -398,7 +398,7 @@ async function runChecks(testFile, doSearch, parseQuery, getCorrections) {
  *   `parseQuery` function exported from the search module; and `getCorrections`, which runs
  *   a search but returns type name corrections instead of results.
  */
-function loadSearchJS(doc_folder, resource_suffix) {
+async function loadSearchJS(doc_folder, resource_suffix) {
     const searchIndexJs = path.join(doc_folder, "search-index" + resource_suffix + ".js");
     const searchIndex = require(searchIndexJs);
 
@@ -433,7 +433,7 @@ function loadSearchJS(doc_folder, resource_suffix) {
     const staticFiles = path.join(doc_folder, "static.files");
     const searchJs = fs.readdirSync(staticFiles).find(f => f.match(/search.*\.js$/));
     const searchModule = require(path.join(staticFiles, searchJs));
-    searchModule.initSearch(searchIndex.searchIndex);
+    await searchModule.initSearch(searchIndex.searchIndex);
     const docSearch = searchModule.docSearch;
     return {
         doSearch: async function(queryStr, filterCrate, currentCrate) {
@@ -549,7 +549,7 @@ async function main(argv) {
         return 1;
     }
 
-    const parseAndSearch = loadSearchJS(
+    const parseAndSearch = await loadSearchJS(
         opts["doc_folder"],
         opts["resource_suffix"],
     );