diff --git a/Cargo.lock b/Cargo.lock index d2b96e6d10270..87e37f50e4b91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5274,8 +5274,7 @@ dependencies = [ [[package]] name = "stringdex" version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18b3bd4f10d15ef859c40291769f0d85209de6b0f1c30713ff9cdf45ac43ea36" +source = "git+https://gitlab.com/yotamofek/stringdex?rev=a1812aefdb3dce2bb85d9ec52a6e53d8eefa3c4b#a1812aefdb3dce2bb85d9ec52a6e53d8eefa3c4b" dependencies = [ "stacker", ] diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index 63412e2b9373d..a6bf39c524968 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -21,7 +21,7 @@ rustdoc-json-types = { path = "../rustdoc-json-types" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" smallvec = "1.8.1" -stringdex = "=0.0.2" +stringdex = { git = "https://gitlab.com/yotamofek/stringdex", rev = "a1812aefdb3dce2bb85d9ec52a6e53d8eefa3c4b" } tempfile = "3" threadpool = "1.8.1" tracing = "0.1" diff --git a/src/librustdoc/html/render/search_index.rs b/src/librustdoc/html/render/search_index.rs index 9c072eed51aec..3514c517d9134 100644 --- a/src/librustdoc/html/render/search_index.rs +++ b/src/librustdoc/html/render/search_index.rs @@ -3,7 +3,9 @@ mod serde; use std::collections::BTreeSet; use std::collections::hash_map::Entry; +use std::io; use std::path::Path; +use std::string::FromUtf8Error; use ::serde::de::{self, Deserializer, Error as _}; use ::serde::ser::{SerializeSeq, Serializer}; @@ -95,21 +97,22 @@ impl SerializedSearchIndex { ) -> Result<(), Error> { let root_path = doc_root.join(format!("search.index/root{resource_suffix}.js")); let column_path = doc_root.join(format!("search.index/{column_name}/")); + + let mut consume = |_, cell: &[u8]| { + column.push(String::from_utf8(cell.to_vec())?); + Ok::<_, FromUtf8Error>(()) + }; + stringdex_internals::read_data_from_disk_column( root_path, column_name.as_bytes(), column_path.clone(), - &mut |_id, item| { - column.push(String::from_utf8(item.to_vec())?); - Ok(()) - }, - ) - .map_err( - |error: stringdex_internals::ReadDataError>| Error { - file: column_path, - error: format!("failed to read column from disk: {error}"), - }, + &mut consume, ) + .map_err(|error| Error { + file: column_path, + error: format!("failed to read column from disk: {error}"), + }) } fn perform_read_serde( resource_suffix: &str, @@ -119,25 +122,26 @@ impl SerializedSearchIndex { ) -> Result<(), Error> { let root_path = doc_root.join(format!("search.index/root{resource_suffix}.js")); let column_path = doc_root.join(format!("search.index/{column_name}/")); + + let mut consume = |_, cell: &[u8]| { + if cell.is_empty() { + column.push(None); + } else { + column.push(Some(serde_json::from_slice(cell)?)); + } + Ok::<_, serde_json::Error>(()) + }; + stringdex_internals::read_data_from_disk_column( root_path, column_name.as_bytes(), column_path.clone(), - &mut |_id, item| { - if item.is_empty() { - column.push(None); - } else { - column.push(Some(serde_json::from_slice(item)?)); - } - Ok(()) - }, - ) - .map_err( - |error: stringdex_internals::ReadDataError>| Error { - file: column_path, - error: format!("failed to read column from disk: {error}"), - }, + &mut consume, ) + .map_err(|error| Error { + file: column_path, + error: format!("failed to read column from disk: {error}"), + }) } fn perform_read_postings( resource_suffix: &str, @@ -147,23 +151,28 @@ impl SerializedSearchIndex { ) -> Result<(), Error> { let root_path = doc_root.join(format!("search.index/root{resource_suffix}.js")); let column_path = doc_root.join(format!("search.index/{column_name}/")); + + fn consumer( + column: &mut Vec>>, + ) -> impl FnMut(u32, &[u8]) -> io::Result<()> { + |_, cell| { + let mut postings = Vec::new(); + encode::read_postings_from_string(&mut postings, cell); + column.push(postings); + Ok(()) + } + } + stringdex_internals::read_data_from_disk_column( root_path, column_name.as_bytes(), column_path.clone(), - &mut |_id, buf| { - let mut postings = Vec::new(); - encode::read_postings_from_string(&mut postings, buf); - column.push(postings); - Ok(()) - }, - ) - .map_err( - |error: stringdex_internals::ReadDataError>| Error { - file: column_path, - error: format!("failed to read column from disk: {error}"), - }, + &mut consumer(column), ) + .map_err(|error| Error { + file: column_path, + error: format!("failed to read column from disk: {error}"), + }) } assert_eq!(names.len(), path_data.len()); @@ -1055,12 +1064,12 @@ impl Serialize for TypeData { let mut buf = Vec::new(); encode::write_postings_to_string(&self.inverted_function_inputs_index, &mut buf); let mut serialized_result = Vec::new(); - stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result); + stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result).unwrap(); seq.serialize_element(&str::from_utf8(&serialized_result).unwrap())?; buf.clear(); serialized_result.clear(); encode::write_postings_to_string(&self.inverted_function_output_index, &mut buf); - stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result); + stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result).unwrap(); seq.serialize_element(&str::from_utf8(&serialized_result).unwrap())?; if self.search_unbox { seq.serialize_element(&1)?; diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index 19c773d12f7fa..a932fe7122d9a 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -11,6 +11,8 @@ const ALLOWED_SOURCES: &[&str] = &[ r#""registry+https://github.com/rust-lang/crates.io-index""#, // This is `rust_team_data` used by `site` in src/tools/rustc-perf, r#""git+https://github.com/rust-lang/team#a5260e76d3aa894c64c56e6ddc8545b9a98043ec""#, + // TMP: + r#""git+https://gitlab.com/yotamofek/stringdex?rev=a1812aefdb3dce2bb85d9ec52a6e53d8eefa3c4b#a1812aefdb3dce2bb85d9ec52a6e53d8eefa3c4b""#, ]; /// Checks for external package sources. `root` is the path to the directory that contains the