Auto merge of #147918 - yotamofek:pr/stringdex-fork, r=<try>

[PERF] see if my fork of `stringdex` affects perf

Author: rust-bors[bot]

Cargo.lock

@@ -5274,8 +5274,7 @@ dependencies = [
 [[package]]
 name = "stringdex"
 version = "0.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18b3bd4f10d15ef859c40291769f0d85209de6b0f1c30713ff9cdf45ac43ea36"
+source = "git+https://gitlab.com/yotamofek/stringdex?rev=61a1cda4f942e6a6773e4b969aff7b26903a31f8#61a1cda4f942e6a6773e4b969aff7b26903a31f8"
 dependencies = [
  "stacker",
 ]

src/librustdoc/Cargo.toml

@@ -21,7 +21,7 @@ rustdoc-json-types = { path = "../rustdoc-json-types" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 smallvec = "1.8.1"
-stringdex = "=0.0.2"
+stringdex = { git = "https://gitlab.com/yotamofek/stringdex", rev = "61a1cda4f942e6a6773e4b969aff7b26903a31f8" }
 tempfile = "3"
 threadpool = "1.8.1"
 tracing = "0.1"

src/librustdoc/html/render/search_index.rs

@@ -3,7 +3,9 @@ mod serde;
 
 use std::collections::BTreeSet;
 use std::collections::hash_map::Entry;
+use std::io;
 use std::path::Path;
+use std::string::FromUtf8Error;
 
 use ::serde::de::{self, Deserializer, Error as _};
 use ::serde::ser::{SerializeSeq, Serializer};
@@ -95,21 +97,28 @@ impl SerializedSearchIndex {
         ) -> Result<(), Error> {
             let root_path = doc_root.join(format!("search.index/root{resource_suffix}.js"));
             let column_path = doc_root.join(format!("search.index/{column_name}/"));
+
+            struct Consumer<'col>(&'col mut Vec<String>);
+
+            impl<'col> stringdex_internals::Consumer for Consumer<'col> {
+                type Err = FromUtf8Error;
+
+                fn consume(&mut self, _id: u32, cell: &[u8]) -> Result<(), Self::Err> {
+                    self.0.push(String::from_utf8(cell.to_vec())?);
+                    Ok(())
+                }
+            }
+
             stringdex_internals::read_data_from_disk_column(
                 root_path,
                 column_name.as_bytes(),
                 column_path.clone(),
-                &mut |_id, item| {
-                    column.push(String::from_utf8(item.to_vec())?);
-                    Ok(())
-                },
-            )
-            .map_err(
-                |error: stringdex_internals::ReadDataError<Box<dyn std::error::Error>>| Error {
-                    file: column_path,
-                    error: format!("failed to read column from disk: {error}"),
-                },
+                &mut Consumer(column),
             )
+            .map_err(|error| Error {
+                file: column_path,
+                error: format!("failed to read column from disk: {error}"),
+            })
         }
         fn perform_read_serde(
             resource_suffix: &str,
@@ -119,25 +128,35 @@ impl SerializedSearchIndex {
         ) -> Result<(), Error> {
             let root_path = doc_root.join(format!("search.index/root{resource_suffix}.js"));
             let column_path = doc_root.join(format!("search.index/{column_name}/"));
+
+            struct Consumer<'col, T>(&'col mut Vec<Option<T>>);
+
+            impl<'col, T> stringdex_internals::Consumer for Consumer<'col, T>
+            where
+                T: for<'de> Deserialize<'de> + 'static,
+            {
+                type Err = serde_json::Error;
+
+                fn consume(&mut self, _id: u32, cell: &[u8]) -> Result<(), Self::Err> {
+                    if cell.is_empty() {
+                        self.0.push(None);
+                    } else {
+                        self.0.push(Some(serde_json::from_slice(cell)?));
+                    }
+                    Ok(())
+                }
+            }
+
             stringdex_internals::read_data_from_disk_column(
                 root_path,
                 column_name.as_bytes(),
                 column_path.clone(),
-                &mut |_id, item| {
-                    if item.is_empty() {
-                        column.push(None);
-                    } else {
-                        column.push(Some(serde_json::from_slice(item)?));
-                    }
-                    Ok(())
-                },
-            )
-            .map_err(
-                |error: stringdex_internals::ReadDataError<Box<dyn std::error::Error>>| Error {
-                    file: column_path,
-                    error: format!("failed to read column from disk: {error}"),
-                },
+                &mut Consumer(column),
             )
+            .map_err(|error| Error {
+                file: column_path,
+                error: format!("failed to read column from disk: {error}"),
+            })
         }
         fn perform_read_postings(
             resource_suffix: &str,
@@ -147,23 +166,30 @@ impl SerializedSearchIndex {
         ) -> Result<(), Error> {
             let root_path = doc_root.join(format!("search.index/root{resource_suffix}.js"));
             let column_path = doc_root.join(format!("search.index/{column_name}/"));
+
+            struct Consumer<'col>(&'col mut Vec<Vec<Vec<u32>>>);
+
+            impl<'col> stringdex_internals::Consumer for Consumer<'col> {
+                type Err = io::Error;
+
+                fn consume(&mut self, _id: u32, cell: &[u8]) -> Result<(), Self::Err> {
+                    let mut postings = Vec::new();
+                    encode::read_postings_from_string(&mut postings, cell);
+                    self.0.push(postings);
+                    Ok(())
+                }
+            }
+
             stringdex_internals::read_data_from_disk_column(
                 root_path,
                 column_name.as_bytes(),
                 column_path.clone(),
-                &mut |_id, buf| {
-                    let mut postings = Vec::new();
-                    encode::read_postings_from_string(&mut postings, buf);
-                    column.push(postings);
-                    Ok(())
-                },
-            )
-            .map_err(
-                |error: stringdex_internals::ReadDataError<Box<dyn std::error::Error>>| Error {
-                    file: column_path,
-                    error: format!("failed to read column from disk: {error}"),
-                },
+                &mut Consumer(column),
             )
+            .map_err(|error| Error {
+                file: column_path,
+                error: format!("failed to read column from disk: {error}"),
+            })
         }
 
         assert_eq!(names.len(), path_data.len());
@@ -1059,12 +1085,14 @@ impl Serialize for TypeData {
             let mut buf = Vec::new();
             encode::write_postings_to_string(&self.inverted_function_inputs_index, &mut buf);
             let mut serialized_result = Vec::new();
-            stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result);
+            stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result)
+                .unwrap();
             seq.serialize_element(&str::from_utf8(&serialized_result).unwrap())?;
             buf.clear();
             serialized_result.clear();
             encode::write_postings_to_string(&self.inverted_function_output_index, &mut buf);
-            stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result);
+            stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result)
+                .unwrap();
             seq.serialize_element(&str::from_utf8(&serialized_result).unwrap())?;
             if self.search_unbox {
                 seq.serialize_element(&1)?;

src/tools/tidy/src/extdeps.rs

@@ -11,6 +11,8 @@ const ALLOWED_SOURCES: &[&str] = &[
     r#""registry+https://github.com/rust-lang/crates.io-index""#,
     // This is `rust_team_data` used by `site` in src/tools/rustc-perf,
     r#""git+https://github.com/rust-lang/team#a5260e76d3aa894c64c56e6ddc8545b9a98043ec""#,
+    // TMP:
+    r#""git+https://gitlab.com/yotamofek/stringdex?rev=61a1cda4f942e6a6773e4b969aff7b26903a31f8#61a1cda4f942e6a6773e4b969aff7b26903a31f8""#,
 ];
 
 /// Checks for external package sources. `root` is the path to the directory that contains the