From 25a3fb3c3698705e69bda16849f5d4184b3b5a8c Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 9 Dec 2024 12:40:28 -0500 Subject: [PATCH 01/26] extern_args module to get proper compiler args from Cargo. --- src/book/mod.rs | 12 +++- src/utils/extern_args.rs | 135 +++++++++++++++++++++++++++++++++++++++ src/utils/mod.rs | 1 + 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 src/utils/extern_args.rs diff --git a/src/book/mod.rs b/src/book/mod.rs index b33ec6f00c..c145e0db67 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -31,6 +31,7 @@ use crate::renderer::{CmdRenderer, HtmlHandlebars, MarkdownRenderer, RenderConte use crate::utils; use crate::config::{Config, RustEdition}; +use crate::utils::extern_args::ExternArgs; /// The object used to manage and build a book. pub struct MDBook { @@ -304,6 +305,14 @@ impl MDBook { let (book, _) = self.preprocess_book(&TestRenderer)?; let color_output = std::io::stderr().is_terminal(); + + // get extra args we'll need for rustdoc + // assumes current working directory is project root, eventually + // pick up manifest directory from some config. + + let mut extern_args = ExternArgs::new(); + extern_args.load(&std::env::current_dir()?)?; + let mut failed = false; for item in book.iter() { if let BookItem::Chapter(ref ch) = *item { @@ -332,7 +341,8 @@ impl MDBook { cmd.current_dir(temp_dir.path()) .arg(chapter_path) .arg("--test") - .args(&library_args); + .args(&library_args) // also need --extern for doctest to actually work + .args(extern_args.get_args()); if let Some(edition) = self.config.rust.edition { match edition { diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs new file mode 100644 index 0000000000..80896aaaba --- /dev/null +++ b/src/utils/extern_args.rs @@ -0,0 +1,135 @@ +//! Get "compiler" args from cargo + +use crate::errors::*; +use log::info; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Get the arguments needed to invoke rustc so it can find external crates +/// when invoked by rustdoc to compile doctests. +/// +/// It seems the `-L ` and `--extern =` args are sufficient. +/// +/// Cargo doesn't expose a stable API to get this information. +/// `cargo metadata` does not include the hash suffix in ``. +/// But it does leak when doing a build in verbose mode. +/// So we force a cargo build, capture the console output and parse the args therefrom. +/// +/// Example: +/// ```rust +/// +/// use mdbook::utils::extern_args::ExternArgs; +/// # use mdbook::errors::*; +/// +/// # fn main() -> Result<()> { +/// // Get cargo to say what the compiler args need to be... +/// let proj_root = std::env::current_dir()?; // or other path to `Cargo.toml` +/// let mut extern_args = ExternArgs::new(); +/// extern_args.load(&proj_root)?; +/// +/// // then, when actually invoking rustdoc or some other compiler-like tool... +/// +/// assert!(extern_args.get_args().iter().any(|e| e == "-L")); // args contains "-L".to_string() +/// assert!(extern_args.get_args().iter().any(|e| e == "--extern")); +/// # Ok(()) +/// # } +/// ``` + +#[derive(Debug)] +pub struct ExternArgs { + suffix_args: Vec, +} + +impl ExternArgs { + /// simple constructor + pub fn new() -> Self { + ExternArgs { + suffix_args: vec![], + } + } + + /// Run a `cargo build` to see what args Cargo is using for library paths and extern crates. + /// Touch a source file to ensure something is compiled and the args will be visible. + /// + /// >>>Future research: see whether `cargo check` can be used instead. It emits the `--extern`s + /// with `.rmeta` instead of `.rlib`, and the compiler can't actually use those + /// when compiling a doctest. But perhaps simply changing the file extension would work? + pub fn load(&mut self, proj_root: &Path) -> Result<&Self> { + // touch (change) a file in the project to force check to do something + + for fname in ["lib.rs", "main.rs"] { + let try_path: PathBuf = [&proj_root.to_string_lossy(), "src", fname] + .iter() + .collect(); + let f = File::options().append(true).open(&try_path)?; + f.set_modified(std::time::SystemTime::now())?; + break; + // file should be closed when f goes out of scope at bottom of this loop + } + + let mut cmd = Command::new("cargo"); + cmd.current_dir(&proj_root).arg("build").arg("--verbose"); + + info!("running {:?}", cmd); + let output = cmd.output()?; + + if !output.status.success() { + bail!("Exit status {} from {:?}", output.status, cmd); + } + + let cmd_resp: &str = std::str::from_utf8(&output.stderr)?; + self.parse_response(&cmd_resp)?; + + Ok(self) + } + + /// Parse response stdout+stderr response from `cargo build` + /// into arguments we can use to invoke rustdoc. + /// + /// >>> This parser is broken, doesn't handle arg values with embedded spaces (single quoted). + /// Fortunately, the args we care about (so far) don't have those kinds of values. + pub fn parse_response(&mut self, buf: &str) -> Result<()> { + for l in buf.lines() { + if let Some(_i) = l.find(" Running ") { + let args_seg: &str = l.split('`').skip(1).take(1).collect::>()[0]; // sadly, cargo decorates string with backticks + let mut arg_iter = args_seg.split_whitespace(); + + while let Some(arg) = arg_iter.next() { + match arg { + "-L" | "--library-path" | "--extern" => { + self.suffix_args.push(arg.to_owned()); + self.suffix_args + .push(arg_iter.next().unwrap_or("").to_owned()); + } + _ => {} + } + } + }; + } + + Ok(()) + } + + /// get a list of (-L and --extern) args used to invoke rustdoc. + pub fn get_args(&self) -> Vec { + self.suffix_args.clone() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parse_response_parses_string() -> Result<()> { + let resp = std::fs::read_to_string("tests/t1.txt")?; + let mut ea = ExternArgs::new(); + ea.parse_response(&resp)?; + + let sfx = ea.get_args(); + assert!(sfx.len() > 0); + + Ok(()) + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index a53f79c0e9..cc7f8f97ca 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -3,6 +3,7 @@ pub mod fs; mod string; pub(crate) mod toml_ext; +pub mod extern_args; use crate::errors::Error; use log::error; use once_cell::sync::Lazy; From a166f21d81c31d83bc28de861f179bf21e51a38f Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 9 Dec 2024 19:29:52 -0500 Subject: [PATCH 02/26] Add config.rust.package-dir and fix cargo output parser (maybe not for the last time) --- src/book/mod.rs | 10 +++++----- src/config.rs | 28 ++++++++++++++++++++++++++- src/utils/extern_args.rs | 41 ++++++++++++++++++++++++++++++++++------ 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/src/book/mod.rs b/src/book/mod.rs index c145e0db67..7cb11aec85 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -306,13 +306,13 @@ impl MDBook { let color_output = std::io::stderr().is_terminal(); - // get extra args we'll need for rustdoc - // assumes current working directory is project root, eventually - // pick up manifest directory from some config. + // get extra args we'll need for rustdoc, if config points to a cargo project. let mut extern_args = ExternArgs::new(); - extern_args.load(&std::env::current_dir()?)?; - + if let Some(package_dir) = &self.config.rust.package_dir { + extern_args.load(&package_dir)?; + } + let mut failed = false; for item in book.iter() { if let BookItem::Chapter(ref ch) = *item { diff --git a/src/config.rs b/src/config.rs index b87ad27644..2a1aa01106 100644 --- a/src/config.rs +++ b/src/config.rs @@ -497,6 +497,8 @@ impl Default for BuildConfig { #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] pub struct RustConfig { + /// Path to a Cargo package + pub package_dir: Option, /// Rust edition used in playground pub edition: Option, } @@ -798,6 +800,9 @@ mod tests { create-missing = false use-default-preprocessors = true + [rust] + package-dir = "." + [output.html] theme = "./themedir" default-theme = "rust" @@ -839,7 +844,10 @@ mod tests { use_default_preprocessors: true, extra_watch_dirs: Vec::new(), }; - let rust_should_be = RustConfig { edition: None }; + let rust_should_be = RustConfig { + package_dir: Some(PathBuf::from(".")), + edition: None, + }; let playground_should_be = Playground { editable: true, copyable: true, @@ -918,6 +926,7 @@ mod tests { assert_eq!(got.book, book_should_be); let rust_should_be = RustConfig { + package_dir: None, edition: Some(RustEdition::E2015), }; let got = Config::from_str(src).unwrap(); @@ -937,6 +946,7 @@ mod tests { "#; let rust_should_be = RustConfig { + package_dir: None, edition: Some(RustEdition::E2018), }; @@ -957,6 +967,7 @@ mod tests { "#; let rust_should_be = RustConfig { + package_dir: None, edition: Some(RustEdition::E2021), }; @@ -1356,4 +1367,19 @@ mod tests { false ); } + + + /* todo -- make this test fail, as it should + #[test] + #[should_panic(expected = "Invalid configuration file")] + // invalid key in config file should really generate an error... + fn invalid_rust_setting() { + let src = r#" + [rust] + foo = "bar" + "#; + + Config::from_str(src).unwrap(); + } + */ } diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index 80896aaaba..903c911b4f 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -1,7 +1,7 @@ //! Get "compiler" args from cargo use crate::errors::*; -use log::info; +use log::{info, warn}; use std::fs::File; use std::path::{Path, PathBuf}; use std::process::Command; @@ -29,7 +29,7 @@ use std::process::Command; /// extern_args.load(&proj_root)?; /// /// // then, when actually invoking rustdoc or some other compiler-like tool... -/// +/// /// assert!(extern_args.get_args().iter().any(|e| e == "-L")); // args contains "-L".to_string() /// assert!(extern_args.get_args().iter().any(|e| e == "--extern")); /// # Ok(()) @@ -86,6 +86,7 @@ impl ExternArgs { /// Parse response stdout+stderr response from `cargo build` /// into arguments we can use to invoke rustdoc. + /// Stop at first line that traces a compiler invocation. /// /// >>> This parser is broken, doesn't handle arg values with embedded spaces (single quoted). /// Fortunately, the args we care about (so far) don't have those kinds of values. @@ -105,9 +106,15 @@ impl ExternArgs { _ => {} } } + + return Ok(()); }; } + if self.suffix_args.len() < 1 { + warn!("Couldn't extract --extern args from Cargo, is current directory == cargo project root?"); + } + Ok(()) } @@ -123,12 +130,34 @@ mod test { #[test] fn parse_response_parses_string() -> Result<()> { - let resp = std::fs::read_to_string("tests/t1.txt")?; + let test_str = r###" + Fresh unicode-ident v1.0.14 + Fresh cfg-if v1.0.0 + Fresh memchr v2.7.4 + Fresh autocfg v1.4.0 + Fresh version_check v0.9.5 + --- clip --- + Fresh bytecount v0.6.8 + Fresh leptos_router v0.7.0 + Fresh leptos_meta v0.7.0 + Fresh console_error_panic_hook v0.1.7 + Fresh mdbook-keeper v0.5.0 + Dirty leptos-book v0.1.0 (/home/bobhy/src/localdep/book): the file `src/lib.rs` has changed (1733758773.052514835s, 10h 32m 29s after last build at 1733720824.458358565s) + Compiling leptos-book v0.1.0 (/home/bobhy/src/localdep/book) + Running `/home/bobhy/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc --crate-name leptos_book --edition=2021 src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type cdylib --crate-type rlib --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --check-cfg 'cfg(docsrs)' --check-cfg 'cfg(feature, values("hydrate", "ssr"))' -C metadata=2eec49d479de095c --out-dir /home/bobhy/src/localdep/book/target/debug/deps -C incremental=/home/bobhy/src/localdep/book/target/debug/incremental -L dependency=/home/bobhy/src/localdep/book/target/debug/deps --extern console_error_panic_hook=/home/bobhy/src/localdep/book/target/debug/deps/libconsole_error_panic_hook-d34cf0116774f283.rlib --extern http=/home/bobhy/src/localdep/book/target/debug/deps/libhttp-d4d503240b7a6b18.rlib --extern leptos=/home/bobhy/src/localdep/book/target/debug/deps/libleptos-1dabf2e09ca58f3d.rlib --extern leptos_meta=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_meta-df8ce1704acca063.rlib --extern leptos_router=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_router-df109cd2ee44b2a0.rlib --extern mdbook_keeper_lib=/home/bobhy/src/localdep/book/target/debug/deps/libmdbook_keeper_lib-f4016aaf2c5da5f2.rlib --extern thiserror=/home/bobhy/src/localdep/book/target/debug/deps/libthiserror-acc5435cdf9551fe.rlib --extern wasm_bindgen=/home/bobhy/src/localdep/book/target/debug/deps/libwasm_bindgen-89a7b1dccd9668ae.rlib` + Running `/home/bobhy/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc --crate-name leptos_book --edition=2021 src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --check-cfg 'cfg(docsrs)' --check-cfg 'cfg(feature, values("hydrate", "ssr"))' -C metadata=24fbc99376c5eff3 -C extra-filename=-24fbc99376c5eff3 --out-dir /home/bobhy/src/localdep/book/target/debug/deps -C incremental=/home/bobhy/src/localdep/book/target/debug/incremental -L dependency=/home/bobhy/src/localdep/book/target/debug/deps --extern console_error_panic_hook=/home/bobhy/src/localdep/book/target/debug/deps/libconsole_error_panic_hook-d34cf0116774f283.rlib --extern http=/home/bobhy/src/localdep/book/target/debug/deps/libhttp-d4d503240b7a6b18.rlib --extern leptos=/home/bobhy/src/localdep/book/target/debug/deps/libleptos-1dabf2e09ca58f3d.rlib --extern leptos_book=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_book.rlib --extern leptos_meta=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_meta-df8ce1704acca063.rlib --extern leptos_router=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_router-df109cd2ee44b2a0.rlib --extern mdbook_keeper_lib=/home/bobhy/src/localdep/book/target/debug/deps/libmdbook_keeper_lib-f4016aaf2c5da5f2.rlib --extern thiserror=/home/bobhy/src/localdep/book/target/debug/deps/libthiserror-acc5435cdf9551fe.rlib --extern wasm_bindgen=/home/bobhy/src/localdep/book/target/debug/deps/libwasm_bindgen-89a7b1dccd9668ae.rlib` + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.60s + + "###; + let mut ea = ExternArgs::new(); - ea.parse_response(&resp)?; + ea.parse_response(&test_str)?; + + let args = ea.get_args(); + assert_eq!(18, args.len()); - let sfx = ea.get_args(); - assert!(sfx.len() > 0); + assert_eq!(1, args.iter().filter(|i| *i == "-L").count()); + assert_eq!(8, args.iter().filter(|i| *i == "--extern").count()); Ok(()) } From c92611377662ddf4fdd4ce35c85c4299aed36f4b Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 9 Dec 2024 23:05:08 -0500 Subject: [PATCH 03/26] Force --extern =.rmeta to .rlib. Cargo sometimes tries to overoptimize compile time. --- src/utils/extern_args.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index 903c911b4f..f1dd61b10d 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -98,11 +98,16 @@ impl ExternArgs { while let Some(arg) = arg_iter.next() { match arg { - "-L" | "--library-path" | "--extern" => { + "-L" | "--library-path" => { self.suffix_args.push(arg.to_owned()); self.suffix_args .push(arg_iter.next().unwrap_or("").to_owned()); } + "--extern" => { // needs a hack to force reference to rlib over rmeta + self.suffix_args.push(arg.to_owned()); + self.suffix_args + .push(arg_iter.next().unwrap_or("").replace(".rmeta", ".rlib").to_owned()); + } _ => {} } } From 3eb20b367d9ba811aac695a2a8895669777e807e Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 9 Dec 2024 23:05:51 -0500 Subject: [PATCH 04/26] Modify user guide so code samples can reference external crates. --- guide/book.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/guide/book.toml b/guide/book.toml index 817f8b07b7..0eb58e24d8 100644 --- a/guide/book.toml +++ b/guide/book.toml @@ -6,6 +6,7 @@ language = "en" [rust] edition = "2018" +package-dir = "../" [output.html] smart-punctuation = true From 96c2aee6beee329fe562efe434208b42ce499aaa Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 9 Dec 2024 23:08:17 -0500 Subject: [PATCH 05/26] Update guide to describe configuring book for external crates; Also refactor the `mdBook test` section and add a new "writing doc tests" chapter. --- guide/src/SUMMARY.md | 1 + guide/src/cli/test.md | 34 ++---- guide/src/format/configuration/general.md | 8 ++ guide/src/guide/README.md | 1 + guide/src/guide/writing.md | 132 ++++++++++++++++++++++ 5 files changed, 153 insertions(+), 23 deletions(-) create mode 100644 guide/src/guide/writing.md diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 974d65fae7..303cc79e2c 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -7,6 +7,7 @@ - [Installation](guide/installation.md) - [Reading Books](guide/reading.md) - [Creating a Book](guide/creating.md) +- [Writing Code Samples](guide/writing.md) # Reference Guide diff --git a/guide/src/cli/test.md b/guide/src/cli/test.md index ba06bd7082..3552b17bb3 100644 --- a/guide/src/cli/test.md +++ b/guide/src/cli/test.md @@ -1,32 +1,16 @@ # The test command -When writing a book, you sometimes need to automate some tests. For example, +When writing a book, you may want to provide some code samples, +and it's important that these be accurate. +For example, [The Rust Programming Book](https://doc.rust-lang.org/stable/book/) uses a lot -of code examples that could get outdated. Therefore it is very important for +of code samples that could get outdated as the language evolves. Therefore it is very important for them to be able to automatically test these code examples. -mdBook supports a `test` command that will run all available tests in a book. At -the moment, only Rust tests are supported. +mdBook supports a `test` command that will run code samples as doc tests for your book. At +the moment, only Rust doc tests are supported. -#### Disable tests on a code block - -rustdoc doesn't test code blocks which contain the `ignore` attribute: - - ```rust,ignore - fn main() {} - ``` - -rustdoc also doesn't test code blocks which specify a language other than Rust: - - ```markdown - **Foo**: _bar_ - ``` - -rustdoc *does* test code blocks which have no language specified: - - ``` - This is going to cause an error! - ``` +For details on writing code samples and runnable code samples in your book, see [Writing](../guide/writing.md). #### Specify a directory @@ -39,6 +23,10 @@ mdbook test path/to/book #### `--library-path` +> Note: This argument doesn't provide sufficient information for current Rust compilers. +Instead, add `package-dir` to your ***book.toml***, as described in [configuration](/format/configuration/general.md#rust-options). + + The `--library-path` (`-L`) option allows you to add directories to the library search path used by `rustdoc` when it builds and tests the examples. Multiple directories can be specified with multiple options (`-L foo -L bar`) or with a diff --git a/guide/src/format/configuration/general.md b/guide/src/format/configuration/general.md index 40a4570132..cabe4592da 100644 --- a/guide/src/format/configuration/general.md +++ b/guide/src/format/configuration/general.md @@ -68,9 +68,16 @@ integration. ```toml [rust] +package-dir = "folder/for/Cargo.toml" edition = "2015" # the default edition for code blocks ``` +- **package-dir**: Folder containing a Cargo package whose targets and dependencies +you want to use in your book's code samples. +It must be specified if you want to test code samples with `use` statements, even if +there is a `Cargo.toml` in the folder containing the `book.toml`. +This can be a relative path, relative to the folder containing `book.toml`. + - **edition**: Rust edition to use by default for the code snippets. Default is `"2015"`. Individual code blocks can be controlled with the `edition2015`, `edition2018` or `edition2021` annotations, such as: @@ -82,6 +89,7 @@ edition = "2015" # the default edition for code blocks ``` ~~~ + ### Build options This controls the build process of your book. diff --git a/guide/src/guide/README.md b/guide/src/guide/README.md index 90deb10e74..31fed08baf 100644 --- a/guide/src/guide/README.md +++ b/guide/src/guide/README.md @@ -5,3 +5,4 @@ This user guide provides an introduction to basic concepts of using mdBook. - [Installation](installation.md) - [Reading Books](reading.md) - [Creating a Book](creating.md) +- [Writing Code Samples](writing.md) diff --git a/guide/src/guide/writing.md b/guide/src/guide/writing.md new file mode 100644 index 0000000000..4f4cbfc2d0 --- /dev/null +++ b/guide/src/guide/writing.md @@ -0,0 +1,132 @@ +# Writing code samples and documentation tests + +If your book is about software, a short code sample may communicate the point better than many words of explanation. +This section describes how to format samples and, perhaps more importantly, how to verify they compile and run +to ensue they stay aligned with the software APIs they describe. + +Code blocks in your book are passed through mdBook and processed by rustdoc. For more details on structuring codeblocks and running doc tests, +refer to the [rustdoc book](https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html) + +### Code blocks for sample code + +You include a code sample in your book as a markdown fenced code block specifying `rust`, like so: + +`````markdown +```rust +let four = 2 + 2; +assert_eq!(four, 4); +``` +````` + +This displays as: + +```rust +let four = 2 + 2; +assert_eq!(four, 4); +``` + +Rustdoc will wrap this sample in a `fn main() {}` so that it can be compiled and even run by `mdbook test`. + +#### Disable tests on a code block + +rustdoc does not test code blocks which contain the `ignore` attribute: + +`````markdown +```rust,ignore +fn main() {} +This would not compile anyway. +``` +````` + +rustdoc also doesn't test code blocks which specify a language other than Rust: + +`````markdown +```markdown +**Foo**: _bar_ +``` +````` + +rustdoc *does* test code blocks which have no language specified: + +`````markdown +``` +let four = 2 + 2; +assert_eq!(four, 4); +``` +````` + +### Hiding source lines within a sample + +A longer sample may contain sections of boilerplate code that are not relevant to the current section of your book. +You can hide source lines within the code block prefixing them with `#_` +(that is a line starting with `#` followed by a single space), like so: + +`````markdown +```rust +# use std::fs::File; +# use std::io::{Write,Result}; +# fn main() -> Result<()> { +let mut file = File::create("foo.txt")?; +file.write_all(b"Hello, world!")?; +# Ok(()) +# } +``` +````` + +This displays as: + +```rust +# use std::fs::File; +# use std::io::{Write,Result}; +# fn main() -> Result<()> { +let mut file = File::create("foo.txt")?; +file.write_all(b"Hello, world!")?; +# Ok(()) +# } +``` + +Note that the code block displays an "show hidden lines" button in the upper right of the code block (when hovered over). + +Note, too, that the sample provided its own `fn main(){}`, so the `use` statements could be positioned outside it. +When rustdoc sees the sample already provides `fn main`, it does *not* do its own wrapping. + + +### Tests using external crates + +The previous example shows that you can `use` a crate within your sample. +But if the crate is an *external* crate, that is, one declared as a dependency in your +package `Cargo.toml`, rustc (the compiler invoked by rustdoc) needs +`-L` and `--extern` switches in order to compile it. +Cargo does this automatically for `cargo build` and `cargo rustdoc` and mdBook can as well. + +To allow mdBook to determine the correct external crate information, +add `package-dir` to your ***book.toml**, as described in [configuration](/format/configuration/general.md#rust-options). +Note that mdBook runs a `cargo build` for the package to determine correct dependencies. + +This example (borrowed from the `serde` crate documentation) compiles and runs in a properly configured book: + +```rust +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +struct Point { + x: i32, + y: i32, +} + +fn main() { + let point = Point { x: 1, y: 2 }; + + // Convert the Point to a JSON string. + let serialized = serde_json::to_string(&point).unwrap(); + + // Prints serialized = {"x":1,"y":2} + println!("serialized = {}", serialized); + + // Convert the JSON string back to a Point. + let deserialized: Point = serde_json::from_str(&serialized).unwrap(); + + // Prints deserialized = Point { x: 1, y: 2 } + println!("deserialized = {:?}", deserialized); +} +``` From 7b3a1e240c75a482244c4b4f4ad9290293e48d4e Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 9 Dec 2024 23:19:43 -0500 Subject: [PATCH 06/26] Fix CI nits. --- src/utils/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/mod.rs b/src/utils/mod.rs index cc7f8f97ca..4650339723 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,9 +1,9 @@ #![allow(missing_docs)] // FIXME: Document this +pub mod extern_args; pub mod fs; mod string; pub(crate) mod toml_ext; -pub mod extern_args; use crate::errors::Error; use log::error; use once_cell::sync::Lazy; From 138256f656c9eec16f55e7a06ca7c12477febf74 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Tue, 10 Dec 2024 19:55:03 -0500 Subject: [PATCH 07/26] Replace unstable fs::set_modified() with ad-hoc "touch" function. --- src/book/mod.rs | 2 +- src/config.rs | 1 - src/utils/extern_args.rs | 70 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/book/mod.rs b/src/book/mod.rs index 7cb11aec85..98abf28aae 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -312,7 +312,7 @@ impl MDBook { if let Some(package_dir) = &self.config.rust.package_dir { extern_args.load(&package_dir)?; } - + let mut failed = false; for item in book.iter() { if let BookItem::Chapter(ref ch) = *item { diff --git a/src/config.rs b/src/config.rs index 2a1aa01106..5d6bcec99d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1368,7 +1368,6 @@ mod tests { ); } - /* todo -- make this test fail, as it should #[test] #[should_panic(expected = "Invalid configuration file")] diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index f1dd61b10d..afd6785ddd 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -2,7 +2,9 @@ use crate::errors::*; use log::{info, warn}; +use std::fs; use std::fs::File; +use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::process::Command; @@ -62,8 +64,7 @@ impl ExternArgs { let try_path: PathBuf = [&proj_root.to_string_lossy(), "src", fname] .iter() .collect(); - let f = File::options().append(true).open(&try_path)?; - f.set_modified(std::time::SystemTime::now())?; + touch(&try_path)?; break; // file should be closed when f goes out of scope at bottom of this loop } @@ -103,10 +104,16 @@ impl ExternArgs { self.suffix_args .push(arg_iter.next().unwrap_or("").to_owned()); } - "--extern" => { // needs a hack to force reference to rlib over rmeta + "--extern" => { + // needs a hack to force reference to rlib over rmeta self.suffix_args.push(arg.to_owned()); - self.suffix_args - .push(arg_iter.next().unwrap_or("").replace(".rmeta", ".rlib").to_owned()); + self.suffix_args.push( + arg_iter + .next() + .unwrap_or("") + .replace(".rmeta", ".rlib") + .to_owned(), + ); } _ => {} } @@ -129,9 +136,31 @@ impl ExternArgs { } } +// Private "touch" function to update file modification time without changing content. +// needed because [std::fs::set_modified] is unstable in rust 1.74, +// which is currently the MSRV for mdBook. It is available in rust 1.76 onward. + +fn touch(victim: &Path) -> Result<()> { + let curr_content = fs::read(victim).with_context(|| "reading existing file")?; + let mut touchfs = File::options() + .append(true) + .open(victim) + .with_context(|| "opening for touch")?; + + let _len_written = touchfs.write(b"z")?; // write a byte + touchfs.flush().expect("closing"); // close the file + drop(touchfs); // close modified file, hopefully updating modification time + + fs::write(victim, curr_content).with_context(|| "trying to restore old content") +} + #[cfg(test)] mod test { use super::*; + use std::fs; + use std::thread; + use std::time::Duration; + use tempfile; #[test] fn parse_response_parses_string() -> Result<()> { @@ -166,4 +195,35 @@ mod test { Ok(()) } + + #[test] + fn verify_touch() -> Result<()> { + const FILE_CONTENT: &[u8] = + b"I am some random text with crlfs \r\n but also nls \n and terminated with a nl \n"; + const DELAY: Duration = Duration::from_millis(10); // don't hang up tests for too long. + + let temp_dir = tempfile::TempDir::new()?; + let mut victim_path = temp_dir.path().to_owned(); + victim_path.push("workfile.dir"); + fs::write(&victim_path, FILE_CONTENT)?; + let old_md = fs::metadata(&victim_path)?; + thread::sleep(DELAY); + + touch(&victim_path)?; + let new_md = fs::metadata(&victim_path)?; + + let act_content = fs::read(&victim_path)?; + + assert_eq!(FILE_CONTENT, act_content); + assert!( + new_md + .modified() + .expect("getting modified time") + .duration_since(old_md.modified().expect("getting modified time old")) + .expect("system botch") + >= DELAY + ); + + Ok(()) + } } From 3306d207292de3e3604f3ed9c008e4b0019f86a9 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Tue, 10 Dec 2024 21:19:38 -0500 Subject: [PATCH 08/26] Fuzzy up touch file test in case sleep() is not totally precise. --- src/utils/extern_args.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index afd6785ddd..7c65ba2526 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -200,7 +200,7 @@ mod test { fn verify_touch() -> Result<()> { const FILE_CONTENT: &[u8] = b"I am some random text with crlfs \r\n but also nls \n and terminated with a nl \n"; - const DELAY: Duration = Duration::from_millis(10); // don't hang up tests for too long. + const DELAY: Duration = Duration::from_millis(20); // don't hang up tests for too long, but maybe 10ms is too short? let temp_dir = tempfile::TempDir::new()?; let mut victim_path = temp_dir.path().to_owned(); @@ -215,15 +215,20 @@ mod test { let act_content = fs::read(&victim_path)?; assert_eq!(FILE_CONTENT, act_content); + let tdif = new_md + .modified() + .expect("getting modified time new") + .duration_since(old_md.modified().expect("getting modified time old")) + .expect("system time botch"); + // can't expect sleep 20ms to actually delay exactly that -- + // but the test is to verify that `touch` made the file look any newer. + // Give ourselves 50% slop under what we were aiming for and call it good enough. assert!( - new_md - .modified() - .expect("getting modified time") - .duration_since(old_md.modified().expect("getting modified time old")) - .expect("system botch") - >= DELAY + tdif >= (DELAY / 2), + "verify_touch: expected {:?}, actual {:?}", + DELAY, + tdif ); - Ok(()) } } From d9d1f35a256b95e124eedb005392d867ab9b115e Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Wed, 11 Dec 2024 21:00:05 -0500 Subject: [PATCH 09/26] trigger rebuild if project has main.rs (but broken for other binary targets) --- src/utils/extern_args.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index 7c65ba2526..d436f5668e 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -64,11 +64,17 @@ impl ExternArgs { let try_path: PathBuf = [&proj_root.to_string_lossy(), "src", fname] .iter() .collect(); - touch(&try_path)?; - break; - // file should be closed when f goes out of scope at bottom of this loop + if try_path.exists() { + touch(&try_path)?; + self.run_cargo(proj_root)?; + return Ok(self); + // file should be closed when f goes out of scope at bottom of this loop + } } + bail!("Couldn't find source target in project {:?}", proj_root) + } + fn run_cargo(&mut self, proj_root: &Path) -> Result<&Self> { let mut cmd = Command::new("cargo"); cmd.current_dir(&proj_root).arg("build").arg("--verbose"); @@ -76,7 +82,12 @@ impl ExternArgs { let output = cmd.output()?; if !output.status.success() { - bail!("Exit status {} from {:?}", output.status, cmd); + bail!( + "Exit status {} from {:?}\nMessage:\n{:?}", + output.status, + cmd, + std::string::String::from_utf8_lossy(&output.stderr) + ); } let cmd_resp: &str = std::str::from_utf8(&output.stderr)?; From 610405136803d03340ca9c890debc903be81a11f Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sat, 14 Dec 2024 01:07:28 -0500 Subject: [PATCH 10/26] Make guide into workspace child so it can share same dependencies with bin. --- Cargo.toml | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 04063a45a6..ecdbb55806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,17 @@ [workspace] -members = [".", "examples/remove-emphasis/mdbook-remove-emphasis"] +members = [".", "examples/remove-emphasis/mdbook-remove-emphasis", "guide"] + +[workspace.dependencies] +anyhow = "1.0.71" +clap = { version = "4.3.12", features = ["cargo", "wrap_help"] } +mdbook = { path = "." } +pulldown-cmark = { version = "0.12.2", default-features = false, features = [ + "html", +] } # Do not update, part of the public api. +serde = { version = "1.0.163", features = ["derive"] } +serde_json = "1.0.96" +semver = "1.0.17" + [package] name = "mdbook" @@ -7,7 +19,7 @@ version = "0.4.43" authors = [ "Mathieu David ", "Michael-F-Bryan ", - "Matt Ickstadt " + "Matt Ickstadt ", ] documentation = "https://rust-lang.github.io/mdBook/index.html" edition = "2021" @@ -20,23 +32,24 @@ description = "Creates a book from markdown files" rust-version = "1.74" [dependencies] -anyhow = "1.0.71" +anyhow.workspace = true chrono = { version = "0.4.24", default-features = false, features = ["clock"] } -clap = { version = "4.3.12", features = ["cargo", "wrap_help"] } +clap.workspace = true clap_complete = "4.3.2" +cargo-manifest = "0.17.0" once_cell = "1.17.1" env_logger = "0.11.1" handlebars = "6.0" log = "0.4.17" memchr = "2.5.0" opener = "0.7.0" -pulldown-cmark = { version = "0.10.0", default-features = false, features = ["html"] } # Do not update, part of the public api. +pulldown-cmark.workspace = true regex = "1.8.1" -serde = { version = "1.0.163", features = ["derive"] } -serde_json = "1.0.96" +serde.workspace = true +serde_json.workspace = true shlex = "1.3.0" tempfile = "3.4.0" -toml = "0.5.11" # Do not update, see https://github.com/rust-lang/mdBook/issues/2037 +toml = "0.5.11" # Do not update, see https://github.com/rust-lang/mdBook/issues/2037 topological-sort = "0.2.2" # Watch feature @@ -59,7 +72,7 @@ ammonia = { version = "4.0.0", optional = true } assert_cmd = "2.0.11" predicates = "3.0.3" select = "0.6.0" -semver = "1.0.17" +semver.workspace = true pretty_assertions = "1.3.0" walkdir = "2.3.3" From cb77a8933308c6e98613807913a198d27bca712c Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sat, 14 Dec 2024 01:10:07 -0500 Subject: [PATCH 11/26] When running build, pull dependencies from the build of the doctest crate --- guide/book.toml | 4 +- src/book/mod.rs | 2 +- src/utils/extern_args.rs | 173 ++++++++++++++++++++++++++++----------- 3 files changed, 126 insertions(+), 53 deletions(-) diff --git a/guide/book.toml b/guide/book.toml index 0eb58e24d8..de050b3139 100644 --- a/guide/book.toml +++ b/guide/book.toml @@ -5,8 +5,8 @@ authors = ["Mathieu David", "Michael-F-Bryan"] language = "en" [rust] -edition = "2018" -package-dir = "../" +## not needed, and will cause an error, if using Cargo.toml: edition = "2021" +package-dir = "." [output.html] smart-punctuation = true diff --git a/src/book/mod.rs b/src/book/mod.rs index 98abf28aae..78be6df4e1 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -237,7 +237,7 @@ impl MDBook { .chapter_titles .extend(preprocess_ctx.chapter_titles.borrow_mut().drain()); - info!("Running the {} backend", renderer.name()); + debug!("Running the {} backend", renderer.name()); renderer .render(&render_context) .with_context(|| "Rendering failed") diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index d436f5668e..b940090994 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -1,7 +1,8 @@ //! Get "compiler" args from cargo use crate::errors::*; -use log::{info, warn}; +use cargo_manifest::{Edition, Manifest, MaybeInherited::Local}; +use log::{debug, info}; use std::fs; use std::fs::File; use std::io::prelude::*; @@ -40,45 +41,73 @@ use std::process::Command; #[derive(Debug)] pub struct ExternArgs { - suffix_args: Vec, + edition: String, + crate_name: String, + lib_list: Vec, + extern_list: Vec, } impl ExternArgs { /// simple constructor pub fn new() -> Self { ExternArgs { - suffix_args: vec![], + edition: String::default(), + crate_name: String::default(), + lib_list: vec![], + extern_list: vec![], } } /// Run a `cargo build` to see what args Cargo is using for library paths and extern crates. - /// Touch a source file to ensure something is compiled and the args will be visible. - /// - /// >>>Future research: see whether `cargo check` can be used instead. It emits the `--extern`s - /// with `.rmeta` instead of `.rlib`, and the compiler can't actually use those - /// when compiling a doctest. But perhaps simply changing the file extension would work? + /// Touch a source file in the crate to ensure something is compiled and the args will be visible. + pub fn load(&mut self, proj_root: &Path) -> Result<&Self> { + // find Cargo.toml and determine the package name and lib or bin source file. + let cargo_path = proj_root.join("Cargo.toml"); + let mut manifest = Manifest::from_path(&cargo_path)?; + manifest.complete_from_path(proj_root)?; // try real hard to determine bin or lib + let package = manifest + .package + .expect("doctest Cargo.toml must include a [package] section"); + + self.crate_name = package.name.replace('-', "_"); // maybe cargo shouldn't allow packages to include non-identifier characters? + // in any case, this won't work when default crate doesn't have package name (which I'm sure cargo allows somehow or another) + self.edition = if let Some(Local(edition)) = package.edition { + my_display_edition(edition) + } else { + "2015".to_owned() // and good luck to you, sir! + }; + + debug!( + "parsed from manifest: name: {}, edition: {}", + self.crate_name, + format!("{:?}", self.edition) + ); + // touch (change) a file in the project to force check to do something + // I haven't figured out how to determine bin or lib source file from cargo, fall back on heuristics here. - for fname in ["lib.rs", "main.rs"] { - let try_path: PathBuf = [&proj_root.to_string_lossy(), "src", fname] - .iter() - .collect(); + for fname in ["main.rs", "lib.rs"] { + let try_path: PathBuf = proj_root.join("src").join(fname); if try_path.exists() { touch(&try_path)?; - self.run_cargo(proj_root)?; + self.run_cargo(proj_root, &cargo_path)?; return Ok(self); // file should be closed when f goes out of scope at bottom of this loop } } - bail!("Couldn't find source target in project {:?}", proj_root) + bail!("Couldn't find lib or bin source in project {:?}", proj_root) } - fn run_cargo(&mut self, proj_root: &Path) -> Result<&Self> { + fn run_cargo(&mut self, proj_root: &Path, manifest_path: &Path) -> Result<&Self> { let mut cmd = Command::new("cargo"); - cmd.current_dir(&proj_root).arg("build").arg("--verbose"); - + cmd.current_dir(&proj_root) + .arg("build") + .arg("--verbose") + .arg("--manifest-path") + .arg(manifest_path); info!("running {:?}", cmd); + let output = cmd.output()?; if !output.status.success() { @@ -90,63 +119,107 @@ impl ExternArgs { ); } + //ultimatedebug std::fs::write(proj_root.join("mdbook_cargo_out.txt"), &output.stderr)?; + let cmd_resp: &str = std::str::from_utf8(&output.stderr)?; - self.parse_response(&cmd_resp)?; + self.parse_response(&self.crate_name.clone(), &cmd_resp)?; Ok(self) } /// Parse response stdout+stderr response from `cargo build` - /// into arguments we can use to invoke rustdoc. - /// Stop at first line that traces a compiler invocation. + /// into arguments we can use to invoke rustdoc (--edition --extern and -L). + /// The response may contain multiple builds, scan for the one that corresponds to the doctest crate. /// - /// >>> This parser is broken, doesn't handle arg values with embedded spaces (single quoted). + /// > This parser is broken, doesn't handle arg values with embedded spaces (single quoted). /// Fortunately, the args we care about (so far) don't have those kinds of values. - pub fn parse_response(&mut self, buf: &str) -> Result<()> { + pub fn parse_response(&mut self, my_crate: &str, buf: &str) -> Result<()> { + let mut builds_ignored = 0; + + let my_cn_arg = format!(" --crate-name {}", my_crate); for l in buf.lines() { if let Some(_i) = l.find(" Running ") { - let args_seg: &str = l.split('`').skip(1).take(1).collect::>()[0]; // sadly, cargo decorates string with backticks - let mut arg_iter = args_seg.split_whitespace(); - - while let Some(arg) = arg_iter.next() { - match arg { - "-L" | "--library-path" => { - self.suffix_args.push(arg.to_owned()); - self.suffix_args - .push(arg_iter.next().unwrap_or("").to_owned()); + if let Some(_cn_pos) = l.find(&my_cn_arg) { + let args_seg: &str = l.split('`').skip(1).take(1).collect::>()[0]; // sadly, cargo decorates string with backticks + let mut arg_iter = args_seg.split_whitespace(); + + while let Some(arg) = arg_iter.next() { + match arg { + "-L" | "--library-path" => { + self.lib_list + .push(arg_iter.next().unwrap_or_default().to_owned()); + } + + "--extern" => { + let mut dep_arg = arg_iter.next().unwrap_or_default().to_owned(); + + // sometimes, build references the.rmeta even though our doctests will require .rlib + // so convert the argument and hope for the best. + // if .rlib is not there when the doctest runs, it will complain. + if dep_arg.ends_with(".rmeta") { + debug!( + "Build referenced {}, converted to .rlib hoping that actual file will be there in time.", + dep_arg); + dep_arg = dep_arg.replace(".rmeta", ".rlib"); + } + self.extern_list.push(dep_arg); + } + + "--crate-name" => { + self.crate_name = arg_iter.next().unwrap_or_default().to_owned(); + } + + _ => { + if let Some((kw, val)) = arg.split_once('=') { + if kw == "--edition" { + self.edition = val.to_owned(); + } + } + } } - "--extern" => { - // needs a hack to force reference to rlib over rmeta - self.suffix_args.push(arg.to_owned()); - self.suffix_args.push( - arg_iter - .next() - .unwrap_or("") - .replace(".rmeta", ".rlib") - .to_owned(), - ); - } - _ => {} } + } else { + builds_ignored += 1; } - - return Ok(()); }; } - if self.suffix_args.len() < 1 { - warn!("Couldn't extract --extern args from Cargo, is current directory == cargo project root?"); + if self.extern_list.len() == 0 || self.lib_list.len() == 0 { + bail!("Couldn't extract -L or --extern args from Cargo, is current directory == cargo project root?"); } + debug!( + "Ignored {} other builds performed in this run", + builds_ignored + ); + Ok(()) } - /// get a list of (-L and --extern) args used to invoke rustdoc. + /// provide the parsed external args used to invoke rustdoc (--edition, -L and --extern). pub fn get_args(&self) -> Vec { - self.suffix_args.clone() + let mut ret_val: Vec = vec!["--edition".to_owned(), self.edition.clone()]; + for i in &self.lib_list { + ret_val.push("-L".to_owned()); + ret_val.push(i.clone()); + } + for j in &self.extern_list { + ret_val.push("--extern".to_owned()); + ret_val.push(j.clone()); + } + ret_val } } +fn my_display_edition(edition: Edition) -> String { + match edition { + Edition::E2015 => "2015", + Edition::E2018 => "2018", + Edition::E2021 => "2021", + Edition::E2024 => "2024", + } + .to_owned() +} // Private "touch" function to update file modification time without changing content. // needed because [std::fs::set_modified] is unstable in rust 1.74, // which is currently the MSRV for mdBook. It is available in rust 1.76 onward. @@ -196,7 +269,7 @@ mod test { "###; let mut ea = ExternArgs::new(); - ea.parse_response(&test_str)?; + ea.parse_response(&test_str, "leptos_book")?; let args = ea.get_args(); assert_eq!(18, args.len()); From 7fa966f4f5bc3b4562c332adf9bb6483a2d5d728 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sat, 14 Dec 2024 01:10:56 -0500 Subject: [PATCH 12/26] Compile, but do not run the preprocessor examples. --- Cargo.lock | 104 ++++++++++++++++++---- guide/Cargo.toml | 14 +++ guide/src/for_developers/preprocessors.md | 5 +- guide/src/lib.rs | 10 +++ 4 files changed, 112 insertions(+), 21 deletions(-) create mode 100644 guide/Cargo.toml create mode 100644 guide/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3fb221bcef..243d3bfd6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -217,6 +217,17 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "cargo-manifest" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2ce2075c35e4b492b93e3d5dd1dd3670de553f15045595daef8164ed9a3751" +dependencies = [ + "serde", + "thiserror", + "toml 0.8.19", +] + [[package]] name = "cc" version = "1.1.36" @@ -1144,6 +1155,7 @@ dependencies = [ "ammonia", "anyhow", "assert_cmd", + "cargo-manifest", "chrono", "clap", "clap_complete", @@ -1161,7 +1173,7 @@ dependencies = [ "pathdiff", "predicates", "pretty_assertions", - "pulldown-cmark 0.10.3", + "pulldown-cmark", "regex", "select", "semver", @@ -1170,18 +1182,32 @@ dependencies = [ "shlex", "tempfile", "tokio", - "toml", + "toml 0.5.11", "topological-sort", "walkdir", "warp", ] +[[package]] +name = "mdbook-book-code-samples" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "mdbook", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "semver", + "serde", + "serde_json", +] + [[package]] name = "mdbook-remove-emphasis" version = "0.1.0" dependencies = [ "mdbook", - "pulldown-cmark 0.12.2", + "pulldown-cmark", "pulldown-cmark-to-cmark", "serde_json", ] @@ -1601,18 +1627,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "pulldown-cmark" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" -dependencies = [ - "bitflags 2.6.0", - "memchr", - "pulldown-cmark-escape", - "unicase", -] - [[package]] name = "pulldown-cmark" version = "0.12.2" @@ -1621,14 +1635,15 @@ checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ "bitflags 2.6.0", "memchr", + "pulldown-cmark-escape", "unicase", ] [[package]] name = "pulldown-cmark-escape" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" +checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "pulldown-cmark-to-cmark" @@ -1636,7 +1651,7 @@ version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e02b63adcb49f2eb675b1694b413b3e9fedbf549dfe2cc98727ad97a0c30650" dependencies = [ - "pulldown-cmark 0.12.2", + "pulldown-cmark", ] [[package]] @@ -1811,6 +1826,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2084,6 +2108,41 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "topological-sort" version = "0.2.2" @@ -2511,6 +2570,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/guide/Cargo.toml b/guide/Cargo.toml new file mode 100644 index 0000000000..48ded4467e --- /dev/null +++ b/guide/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "mdbook-book-code-samples" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +mdbook.workspace = true +pulldown-cmark.workspace = true +pulldown-cmark-to-cmark = "18.0.0" +serde.workspace = true +serde_json.workspace = true +semver.workspace = true diff --git a/guide/src/for_developers/preprocessors.md b/guide/src/for_developers/preprocessors.md index 1455aceb7a..3cd88797fb 100644 --- a/guide/src/for_developers/preprocessors.md +++ b/guide/src/for_developers/preprocessors.md @@ -36,9 +36,8 @@ be adapted for other preprocessors.
Example no-op preprocessor -```rust +```rust,no_run // nop-preprocessors.rs - {{#include ../../../examples/nop-preprocessor.rs}} ```
@@ -67,7 +66,7 @@ translate events back into markdown text. The following code block shows how to remove all emphasis from markdown, without accidentally breaking the document. -```rust +```rust,no_run {{#rustdoc_include ../../../examples/remove-emphasis/mdbook-remove-emphasis/src/main.rs:remove_emphasis}} ``` diff --git a/guide/src/lib.rs b/guide/src/lib.rs new file mode 100644 index 0000000000..1002228190 --- /dev/null +++ b/guide/src/lib.rs @@ -0,0 +1,10 @@ +// no code yet? +#![allow(unused)] +use mdbook; +use serde_json; +use pulldown_cmark; +use pulldown_cmark_to_cmark; + +pub fn marco() { + +} From 1703aa19e0915f723d51463c7bddb000ad8098f1 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sat, 14 Dec 2024 16:35:07 -0500 Subject: [PATCH 13/26] fix broken unit tests --- src/book/mod.rs | 2 +- src/renderer/html_handlebars/search.rs | 1 + src/utils/extern_args.rs | 15 +++++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/book/mod.rs b/src/book/mod.rs index 78be6df4e1..98abf28aae 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -237,7 +237,7 @@ impl MDBook { .chapter_titles .extend(preprocess_ctx.chapter_titles.borrow_mut().drain()); - debug!("Running the {} backend", renderer.name()); + info!("Running the {} backend", renderer.name()); renderer .render(&render_context) .with_context(|| "Rendering failed") diff --git a/src/renderer/html_handlebars/search.rs b/src/renderer/html_handlebars/search.rs index c03eb4f867..31ce20e337 100644 --- a/src/renderer/html_handlebars/search.rs +++ b/src/renderer/html_handlebars/search.rs @@ -206,6 +206,7 @@ fn render_item( body.push_str(&format!(" [{number}] ")); } Event::TaskListMarker(_checked) => {} + Event::InlineMath(_) | Event::DisplayMath(_) => {} } } diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index b940090994..7abadfab6c 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -41,7 +41,7 @@ use std::process::Command; #[derive(Debug)] pub struct ExternArgs { - edition: String, + edition: String, // where default value of "" means arg wasn't specified crate_name: String, lib_list: Vec, extern_list: Vec, @@ -75,7 +75,7 @@ impl ExternArgs { self.edition = if let Some(Local(edition)) = package.edition { my_display_edition(edition) } else { - "2015".to_owned() // and good luck to you, sir! + "".to_owned() // }; debug!( @@ -198,7 +198,11 @@ impl ExternArgs { /// provide the parsed external args used to invoke rustdoc (--edition, -L and --extern). pub fn get_args(&self) -> Vec { - let mut ret_val: Vec = vec!["--edition".to_owned(), self.edition.clone()]; + let mut ret_val: Vec = vec![]; + if self.edition != "" { + ret_val.push("--edition".to_owned()); + ret_val.push(self.edition.clone()); + }; for i in &self.lib_list { ret_val.push("-L".to_owned()); ret_val.push(i.clone()); @@ -263,16 +267,15 @@ mod test { Dirty leptos-book v0.1.0 (/home/bobhy/src/localdep/book): the file `src/lib.rs` has changed (1733758773.052514835s, 10h 32m 29s after last build at 1733720824.458358565s) Compiling leptos-book v0.1.0 (/home/bobhy/src/localdep/book) Running `/home/bobhy/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc --crate-name leptos_book --edition=2021 src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type cdylib --crate-type rlib --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --check-cfg 'cfg(docsrs)' --check-cfg 'cfg(feature, values("hydrate", "ssr"))' -C metadata=2eec49d479de095c --out-dir /home/bobhy/src/localdep/book/target/debug/deps -C incremental=/home/bobhy/src/localdep/book/target/debug/incremental -L dependency=/home/bobhy/src/localdep/book/target/debug/deps --extern console_error_panic_hook=/home/bobhy/src/localdep/book/target/debug/deps/libconsole_error_panic_hook-d34cf0116774f283.rlib --extern http=/home/bobhy/src/localdep/book/target/debug/deps/libhttp-d4d503240b7a6b18.rlib --extern leptos=/home/bobhy/src/localdep/book/target/debug/deps/libleptos-1dabf2e09ca58f3d.rlib --extern leptos_meta=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_meta-df8ce1704acca063.rlib --extern leptos_router=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_router-df109cd2ee44b2a0.rlib --extern mdbook_keeper_lib=/home/bobhy/src/localdep/book/target/debug/deps/libmdbook_keeper_lib-f4016aaf2c5da5f2.rlib --extern thiserror=/home/bobhy/src/localdep/book/target/debug/deps/libthiserror-acc5435cdf9551fe.rlib --extern wasm_bindgen=/home/bobhy/src/localdep/book/target/debug/deps/libwasm_bindgen-89a7b1dccd9668ae.rlib` - Running `/home/bobhy/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc --crate-name leptos_book --edition=2021 src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --check-cfg 'cfg(docsrs)' --check-cfg 'cfg(feature, values("hydrate", "ssr"))' -C metadata=24fbc99376c5eff3 -C extra-filename=-24fbc99376c5eff3 --out-dir /home/bobhy/src/localdep/book/target/debug/deps -C incremental=/home/bobhy/src/localdep/book/target/debug/incremental -L dependency=/home/bobhy/src/localdep/book/target/debug/deps --extern console_error_panic_hook=/home/bobhy/src/localdep/book/target/debug/deps/libconsole_error_panic_hook-d34cf0116774f283.rlib --extern http=/home/bobhy/src/localdep/book/target/debug/deps/libhttp-d4d503240b7a6b18.rlib --extern leptos=/home/bobhy/src/localdep/book/target/debug/deps/libleptos-1dabf2e09ca58f3d.rlib --extern leptos_book=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_book.rlib --extern leptos_meta=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_meta-df8ce1704acca063.rlib --extern leptos_router=/home/bobhy/src/localdep/book/target/debug/deps/libleptos_router-df109cd2ee44b2a0.rlib --extern mdbook_keeper_lib=/home/bobhy/src/localdep/book/target/debug/deps/libmdbook_keeper_lib-f4016aaf2c5da5f2.rlib --extern thiserror=/home/bobhy/src/localdep/book/target/debug/deps/libthiserror-acc5435cdf9551fe.rlib --extern wasm_bindgen=/home/bobhy/src/localdep/book/target/debug/deps/libwasm_bindgen-89a7b1dccd9668ae.rlib` Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.60s "###; let mut ea = ExternArgs::new(); - ea.parse_response(&test_str, "leptos_book")?; + ea.parse_response("leptos_book", &test_str)?; let args = ea.get_args(); - assert_eq!(18, args.len()); + assert_eq!(20, args.len()); assert_eq!(1, args.iter().filter(|i| *i == "-L").count()); assert_eq!(8, args.iter().filter(|i| *i == "--extern").count()); From 9532dbced3b51518ff61f9cd299e2209c1b3138d Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sun, 15 Dec 2024 00:35:58 -0500 Subject: [PATCH 14/26] book.tom.l must configure path to Cargo.toml, not just root folder of project: changed config rust.package-dir to rust.manifest --- guide/book.toml | 2 +- src/book/mod.rs | 4 ++-- src/cmd/test.rs | 2 +- src/config.rs | 14 +++++++------- src/utils/extern_args.rs | 18 +++++++++++------- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/guide/book.toml b/guide/book.toml index de050b3139..32acd675d6 100644 --- a/guide/book.toml +++ b/guide/book.toml @@ -6,7 +6,7 @@ language = "en" [rust] ## not needed, and will cause an error, if using Cargo.toml: edition = "2021" -package-dir = "." +manifest = "Cargo.toml" [output.html] smart-punctuation = true diff --git a/src/book/mod.rs b/src/book/mod.rs index 98abf28aae..f581b26701 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -309,8 +309,8 @@ impl MDBook { // get extra args we'll need for rustdoc, if config points to a cargo project. let mut extern_args = ExternArgs::new(); - if let Some(package_dir) = &self.config.rust.package_dir { - extern_args.load(&package_dir)?; + if let Some(manifest) = &self.config.rust.manifest { + extern_args.load(&manifest)?; } let mut failed = false; diff --git a/src/cmd/test.rs b/src/cmd/test.rs index d41e9ef9eb..4fcbf527f4 100644 --- a/src/cmd/test.rs +++ b/src/cmd/test.rs @@ -28,7 +28,7 @@ pub fn make_subcommand() -> Command { .value_parser(NonEmptyStringValueParser::new()) .action(ArgAction::Append) .help( - "A comma-separated list of directories to add to the crate \ + "[deprecated] A comma-separated list of directories to add to the crate \ search path when building tests", ), ) diff --git a/src/config.rs b/src/config.rs index 5d6bcec99d..dd5657e811 100644 --- a/src/config.rs +++ b/src/config.rs @@ -497,8 +497,8 @@ impl Default for BuildConfig { #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] pub struct RustConfig { - /// Path to a Cargo package - pub package_dir: Option, + /// Path to a Cargo.toml + pub manifest: Option, /// Rust edition used in playground pub edition: Option, } @@ -801,7 +801,7 @@ mod tests { use-default-preprocessors = true [rust] - package-dir = "." + manifest = "./Cargo.toml" [output.html] theme = "./themedir" @@ -845,7 +845,7 @@ mod tests { extra_watch_dirs: Vec::new(), }; let rust_should_be = RustConfig { - package_dir: Some(PathBuf::from(".")), + manifest: Some(PathBuf::from("./Cargo.toml")), edition: None, }; let playground_should_be = Playground { @@ -926,7 +926,7 @@ mod tests { assert_eq!(got.book, book_should_be); let rust_should_be = RustConfig { - package_dir: None, + manifest: None, edition: Some(RustEdition::E2015), }; let got = Config::from_str(src).unwrap(); @@ -946,7 +946,7 @@ mod tests { "#; let rust_should_be = RustConfig { - package_dir: None, + manifest: None, edition: Some(RustEdition::E2018), }; @@ -967,7 +967,7 @@ mod tests { "#; let rust_should_be = RustConfig { - package_dir: None, + manifest: None, edition: Some(RustEdition::E2021), }; diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index 7abadfab6c..07ee1c3478 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -1,6 +1,7 @@ //! Get "compiler" args from cargo use crate::errors::*; +use anyhow::anyhow; use cargo_manifest::{Edition, Manifest, MaybeInherited::Local}; use log::{debug, info}; use std::fs; @@ -27,9 +28,9 @@ use std::process::Command; /// /// # fn main() -> Result<()> { /// // Get cargo to say what the compiler args need to be... -/// let proj_root = std::env::current_dir()?; // or other path to `Cargo.toml` +/// let manifest_file = std::env::current_dir()?.join("Cargo.toml"); // or other path to `Cargo.toml` /// let mut extern_args = ExternArgs::new(); -/// extern_args.load(&proj_root)?; +/// extern_args.load(&manifest_file)?; /// /// // then, when actually invoking rustdoc or some other compiler-like tool... /// @@ -61,11 +62,14 @@ impl ExternArgs { /// Run a `cargo build` to see what args Cargo is using for library paths and extern crates. /// Touch a source file in the crate to ensure something is compiled and the args will be visible. - pub fn load(&mut self, proj_root: &Path) -> Result<&Self> { + pub fn load(&mut self, cargo_path: &Path) -> Result<&Self> { // find Cargo.toml and determine the package name and lib or bin source file. - let cargo_path = proj_root.join("Cargo.toml"); + let proj_root = cargo_path + .canonicalize()? + .parent() + .ok_or(anyhow!("can't find parent of {:?}", cargo_path))?.to_owned(); let mut manifest = Manifest::from_path(&cargo_path)?; - manifest.complete_from_path(proj_root)?; // try real hard to determine bin or lib + manifest.complete_from_path(&proj_root)?; // try real hard to determine bin or lib let package = manifest .package .expect("doctest Cargo.toml must include a [package] section"); @@ -75,7 +79,7 @@ impl ExternArgs { self.edition = if let Some(Local(edition)) = package.edition { my_display_edition(edition) } else { - "".to_owned() // + "".to_owned() // }; debug!( @@ -91,7 +95,7 @@ impl ExternArgs { let try_path: PathBuf = proj_root.join("src").join(fname); if try_path.exists() { touch(&try_path)?; - self.run_cargo(proj_root, &cargo_path)?; + self.run_cargo(&proj_root, &cargo_path)?; return Ok(self); // file should be closed when f goes out of scope at bottom of this loop } From 9cbd0472e1bea251b860056134a891485f18faca Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sun, 15 Dec 2024 00:36:56 -0500 Subject: [PATCH 15/26] Doc changes to cover how to use external crates in doctests (and relation to rustdoc) --- guide/src/SUMMARY.md | 1 - guide/src/cli/test.md | 34 +++--- guide/src/format/configuration/general.md | 53 ++++----- guide/src/format/mdbook.md | 19 ++++ guide/src/guide/writing.md | 132 ---------------------- src/utils/extern_args.rs | 3 +- 6 files changed, 59 insertions(+), 183 deletions(-) delete mode 100644 guide/src/guide/writing.md diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 303cc79e2c..974d65fae7 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -7,7 +7,6 @@ - [Installation](guide/installation.md) - [Reading Books](guide/reading.md) - [Creating a Book](guide/creating.md) -- [Writing Code Samples](guide/writing.md) # Reference Guide diff --git a/guide/src/cli/test.md b/guide/src/cli/test.md index 3552b17bb3..c1743d4656 100644 --- a/guide/src/cli/test.md +++ b/guide/src/cli/test.md @@ -8,9 +8,7 @@ of code samples that could get outdated as the language evolves. Therefore it is them to be able to automatically test these code examples. mdBook supports a `test` command that will run code samples as doc tests for your book. At -the moment, only Rust doc tests are supported. - -For details on writing code samples and runnable code samples in your book, see [Writing](../guide/writing.md). +the moment, only Rust doc tests are supported. #### Specify a directory @@ -21,10 +19,22 @@ instead of the current working directory. mdbook test path/to/book ``` -#### `--library-path` +#### `--dest-dir` + +The `--dest-dir` (`-d`) option allows you to change the output directory for the +book. Relative paths are interpreted relative to the book's root directory. If +not specified it will default to the value of the `build.build-dir` key in +`book.toml`, or to `./book`. + +#### `--chapter` + +The `--chapter` (`-c`) option allows you to test a specific chapter of the +book using the chapter name or the relative path to the chapter. + +#### `--library-path` `[`deprecated`]` -> Note: This argument doesn't provide sufficient information for current Rust compilers. -Instead, add `package-dir` to your ***book.toml***, as described in [configuration](/format/configuration/general.md#rust-options). +> Note: This argument doesn't provide sufficient extern crate information to run doc tests in current Rust compilers. +Instead, add **manifest** to point to a **Cargo.toml** file in your ***book.toml***, as described in [rust configuration](/format/configuration/general.html#rust-options). The `--library-path` (`-L`) option allows you to add directories to the library @@ -41,15 +51,3 @@ mdbook test my-book -L target/debug/deps/ See the `rustdoc` command-line [documentation](https://doc.rust-lang.org/rustdoc/command-line-arguments.html#-l--library-path-where-to-look-for-dependencies) for more information. - -#### `--dest-dir` - -The `--dest-dir` (`-d`) option allows you to change the output directory for the -book. Relative paths are interpreted relative to the book's root directory. If -not specified it will default to the value of the `build.build-dir` key in -`book.toml`, or to `./book`. - -#### `--chapter` - -The `--chapter` (`-c`) option allows you to test a specific chapter of the -book using the chapter name or the relative path to the chapter. diff --git a/guide/src/format/configuration/general.md b/guide/src/format/configuration/general.md index cabe4592da..0963eb0887 100644 --- a/guide/src/format/configuration/general.md +++ b/guide/src/format/configuration/general.md @@ -11,7 +11,7 @@ authors = ["John Doe"] description = "The example book covers examples." [rust] -edition = "2018" +manifest = "./Cargo.toml" [build] build-dir = "my-example-book" @@ -30,7 +30,7 @@ limit-results = 15 ## Supported configuration options -It is important to note that **any** relative path specified in the +> Note: **any** relative path specified in the configuration will always be taken relative from the root of the book where the configuration file is located. @@ -38,6 +38,16 @@ configuration file is located. This is general information about your book. +```toml +[book] +title = "Example book" +authors = ["John Doe", "Jane Doe"] +description = "The example book covers examples." +src = "my-src" # source files in `root/my-src` instead of `root/src` +language = "en" +text-direction = "ltr" +``` + - **title:** The title of the book - **authors:** The author(s) of the book - **description:** A description for the book, which is added as meta @@ -50,17 +60,6 @@ This is general information about your book. - **text-direction**: The direction of text in the book: Left-to-right (LTR) or Right-to-left (RTL). Possible values: `ltr`, `rtl`. When not specified, the text direction is derived from the book's `language` attribute. -**book.toml** -```toml -[book] -title = "Example book" -authors = ["John Doe", "Jane Doe"] -description = "The example book covers examples." -src = "my-src" # the source files will be found in `root/my-src` instead of `root/src` -language = "en" -text-direction = "ltr" -``` - ### Rust options Options for the Rust language, relevant to running tests and playground @@ -68,26 +67,18 @@ integration. ```toml [rust] -package-dir = "folder/for/Cargo.toml" -edition = "2015" # the default edition for code blocks +manifest = "path/for/Cargo.toml" +edition = "2015" # [deprecated] the default edition for code blocks ``` -- **package-dir**: Folder containing a Cargo package whose targets and dependencies -you want to use in your book's code samples. -It must be specified if you want to test code samples with `use` statements, even if -there is a `Cargo.toml` in the folder containing the `book.toml`. -This can be a relative path, relative to the folder containing `book.toml`. - -- **edition**: Rust edition to use by default for the code snippets. Default - is `"2015"`. Individual code blocks can be controlled with the `edition2015`, - `edition2018` or `edition2021` annotations, such as: - - ~~~text - ```rust,edition2015 - // This only works in 2015. - let try = true; - ``` - ~~~ +- **manifest**: Path to a ***Cargo.toml*** file which is used to resolve dependencies of your sample code. mdBook also uses the `package.edition` configured in the cargo project as the default for code snippets in your book. +See [Using External Crates and Dependencies](/format/mdbook.html#using-external-crates-and-dependencies) for details. + +- **edition** `[`deprecated`]`: Rust edition to use by default for the code snippets. +Default is `"2015"`. Individual code blocks can be controlled with the `edition2015`, + `edition2018` or `edition2021` annotations, as described in [Rust code block attributes](/format/mdbook.html#rust-code-block-attributes). + This option is deprecated because it's only useful if your code samples don't depend on external crates or you're not doctest'ing them. In any case, this option cannot be specified if **manifest** is configured. + ### Build options diff --git a/guide/src/format/mdbook.md b/guide/src/format/mdbook.md index 9bb94615ce..15974bcfb0 100644 --- a/guide/src/format/mdbook.md +++ b/guide/src/format/mdbook.md @@ -1,5 +1,9 @@ # mdBook-specific features +# Features for code blocks + +These capabilities primarily affect how the user sees or interacts with code samples in your book and are supported directly by mdBook. Some also affect documentation tests (for which mdBook invokes `rustdoc --test`): this is detailed in the sections below. + ## Hiding code lines There is a feature in mdBook that lets you hide code lines by prepending them with a specific prefix. @@ -306,6 +310,21 @@ And the `editable` attribute will enable the [editor] as described at [Rust code [Rust Playground]: https://play.rust-lang.org/ +## Using external crates and dependencies + +If your code samples depend on external crates, you will probably want to include `use ` statements in the code and want them to resolve and allow documentation tests to run. +To configure this: + +1. Create a ***Cargo.toml*** file with a `[package.dependencies]` section that defines a dependency for each `` you want to use in any sample. If your book is already embedded in an existing Cargo project, you may be able to use the existing project `Cargo.toml`. +2. In your ***book.toml***: + * configure the path to ***Cargo.toml** in `rust.manifest`, as described in [rust configuration](/format/configuration/general.html#rust-options). + * remove `rust.edition` if it is configured. The edition of rust compiler will be as specified in the ***Cargo.toml***. + * Refrain from invoking `mdbook test` with `-L` or `--library-path` argument. This, too, will be inferred from cargo project configuration + +# Features for general content + +These can be used in markdown text (outside code blocks). + ## Controlling page \ A chapter can set a \ that is different from its entry in the table of diff --git a/guide/src/guide/writing.md b/guide/src/guide/writing.md deleted file mode 100644 index 4f4cbfc2d0..0000000000 --- a/guide/src/guide/writing.md +++ /dev/null @@ -1,132 +0,0 @@ -# Writing code samples and documentation tests - -If your book is about software, a short code sample may communicate the point better than many words of explanation. -This section describes how to format samples and, perhaps more importantly, how to verify they compile and run -to ensue they stay aligned with the software APIs they describe. - -Code blocks in your book are passed through mdBook and processed by rustdoc. For more details on structuring codeblocks and running doc tests, -refer to the [rustdoc book](https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html) - -### Code blocks for sample code - -You include a code sample in your book as a markdown fenced code block specifying `rust`, like so: - -`````markdown -```rust -let four = 2 + 2; -assert_eq!(four, 4); -``` -````` - -This displays as: - -```rust -let four = 2 + 2; -assert_eq!(four, 4); -``` - -Rustdoc will wrap this sample in a `fn main() {}` so that it can be compiled and even run by `mdbook test`. - -#### Disable tests on a code block - -rustdoc does not test code blocks which contain the `ignore` attribute: - -`````markdown -```rust,ignore -fn main() {} -This would not compile anyway. -``` -````` - -rustdoc also doesn't test code blocks which specify a language other than Rust: - -`````markdown -```markdown -**Foo**: _bar_ -``` -````` - -rustdoc *does* test code blocks which have no language specified: - -`````markdown -``` -let four = 2 + 2; -assert_eq!(four, 4); -``` -````` - -### Hiding source lines within a sample - -A longer sample may contain sections of boilerplate code that are not relevant to the current section of your book. -You can hide source lines within the code block prefixing them with `#_` -(that is a line starting with `#` followed by a single space), like so: - -`````markdown -```rust -# use std::fs::File; -# use std::io::{Write,Result}; -# fn main() -> Result<()> { -let mut file = File::create("foo.txt")?; -file.write_all(b"Hello, world!")?; -# Ok(()) -# } -``` -````` - -This displays as: - -```rust -# use std::fs::File; -# use std::io::{Write,Result}; -# fn main() -> Result<()> { -let mut file = File::create("foo.txt")?; -file.write_all(b"Hello, world!")?; -# Ok(()) -# } -``` - -Note that the code block displays an "show hidden lines" button in the upper right of the code block (when hovered over). - -Note, too, that the sample provided its own `fn main(){}`, so the `use` statements could be positioned outside it. -When rustdoc sees the sample already provides `fn main`, it does *not* do its own wrapping. - - -### Tests using external crates - -The previous example shows that you can `use` a crate within your sample. -But if the crate is an *external* crate, that is, one declared as a dependency in your -package `Cargo.toml`, rustc (the compiler invoked by rustdoc) needs -`-L` and `--extern` switches in order to compile it. -Cargo does this automatically for `cargo build` and `cargo rustdoc` and mdBook can as well. - -To allow mdBook to determine the correct external crate information, -add `package-dir` to your ***book.toml**, as described in [configuration](/format/configuration/general.md#rust-options). -Note that mdBook runs a `cargo build` for the package to determine correct dependencies. - -This example (borrowed from the `serde` crate documentation) compiles and runs in a properly configured book: - -```rust -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, Debug)] -struct Point { - x: i32, - y: i32, -} - -fn main() { - let point = Point { x: 1, y: 2 }; - - // Convert the Point to a JSON string. - let serialized = serde_json::to_string(&point).unwrap(); - - // Prints serialized = {"x":1,"y":2} - println!("serialized = {}", serialized); - - // Convert the JSON string back to a Point. - let deserialized: Point = serde_json::from_str(&serialized).unwrap(); - - // Prints deserialized = Point { x: 1, y: 2 } - println!("deserialized = {:?}", deserialized); -} -``` diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index 07ee1c3478..a5eb88c578 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -67,7 +67,8 @@ impl ExternArgs { let proj_root = cargo_path .canonicalize()? .parent() - .ok_or(anyhow!("can't find parent of {:?}", cargo_path))?.to_owned(); + .ok_or(anyhow!("can't find parent of {:?}", cargo_path))? + .to_owned(); let mut manifest = Manifest::from_path(&cargo_path)?; manifest.complete_from_path(&proj_root)?; // try real hard to determine bin or lib let package = manifest From 9c5dec2795987e6dc2c7a9a2e292a852a5812fe0 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sun, 15 Dec 2024 00:41:04 -0500 Subject: [PATCH 16/26] fix fmt not caught locally --- guide/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/guide/src/lib.rs b/guide/src/lib.rs index 1002228190..cefd0a0e41 100644 --- a/guide/src/lib.rs +++ b/guide/src/lib.rs @@ -1,10 +1,8 @@ // no code yet? #![allow(unused)] use mdbook; -use serde_json; use pulldown_cmark; use pulldown_cmark_to_cmark; +use serde_json; -pub fn marco() { - -} +pub fn marco() {} From 658221c1700b1f52ef9a082a325b825d5929525e Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 16 Dec 2024 12:33:51 -0500 Subject: [PATCH 17/26] Document deprecating `mdbook test -L` and preferring [rust.manifest] over [rust.edition]. --- guide/src/cli/test.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/guide/src/cli/test.md b/guide/src/cli/test.md index c1743d4656..4549fe4d5e 100644 --- a/guide/src/cli/test.md +++ b/guide/src/cli/test.md @@ -1,14 +1,14 @@ # The test command When writing a book, you may want to provide some code samples, -and it's important that these be accurate. +and it's important that these be kept accurate as your software API evolves. For example, [The Rust Programming Book](https://doc.rust-lang.org/stable/book/) uses a lot -of code samples that could get outdated as the language evolves. Therefore it is very important for -them to be able to automatically test these code examples. +of code samples that could become outdated as the language evolves. -mdBook supports a `test` command that will run code samples as doc tests for your book. At -the moment, only Rust doc tests are supported. +MdBook supports a `test` command which runs code samples in your book as doc tests to verify they +will compile, and, optionally, run correctly. +At the moment, mdBook only supports doc *tests* written in Rust, although code samples can be written and *displayed* in many programming languages. #### Specify a directory @@ -33,8 +33,8 @@ book using the chapter name or the relative path to the chapter. #### `--library-path` `[`deprecated`]` -> Note: This argument doesn't provide sufficient extern crate information to run doc tests in current Rust compilers. -Instead, add **manifest** to point to a **Cargo.toml** file in your ***book.toml***, as described in [rust configuration](/format/configuration/general.html#rust-options). +***Note*** This argument is deprecated. Since Rust edition 2018, the compiler needs an explicit `--extern` argument for each external crate used in a doc test, it no longer simply scans the library path for likely-looking crates. +New projects should list external crates as dependencies in a **Cargo.toml** file and reference that file in your ***book.toml***, as described in [rust configuration](/format/configuration/general.html#rust-options). The `--library-path` (`-L`) option allows you to add directories to the library From a3fc58b88dd63240cad9c6b6c6b0f379d1253699 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 16 Dec 2024 20:41:31 -0500 Subject: [PATCH 18/26] fix `mdbook test ; make chap Alternate Backends pass doctest. --- guide/src/for_developers/backends.md | 25 ++++++++++++++----------- src/book/mod.rs | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/guide/src/for_developers/backends.md b/guide/src/for_developers/backends.md index 72f8263eb5..6f3062075d 100644 --- a/guide/src/for_developers/backends.md +++ b/guide/src/for_developers/backends.md @@ -31,9 +31,9 @@ a [`RenderContext::from_json()`] constructor which will load a `RenderContext`. This is all the boilerplate necessary for our backend to load the book. -```rust +```rust,should_panic +# // this sample panics because it can't open stdin // src/main.rs -extern crate mdbook; use std::io; use mdbook::renderer::RenderContext; @@ -55,14 +55,18 @@ fn main() { ## Inspecting the Book -Now our backend has a copy of the book, lets count how many words are in each +Now our backend has a copy of the book, let's count how many words are in each chapter! Because the `RenderContext` contains a [`Book`] field (`book`), and a `Book` has the [`Book::iter()`] method for iterating over all items in a `Book`, this step turns out to be just as easy as the first. -```rust +```rust,should_panic +# // this sample panics because it can't open stdin +use std::io; +use mdbook::renderer::RenderContext; +use mdbook::book::{BookItem, Chapter}; fn main() { let mut stdin = io::stdin(); @@ -174,26 +178,25 @@ deserializing to some arbitrary type `T`. To implement this, we'll create our own serializable `WordcountConfig` struct which will encapsulate all configuration for this backend. -First add `serde` and `serde_derive` to your `Cargo.toml`, +First add `serde` to your `Cargo.toml`, -``` -$ cargo add serde serde_derive +```shell +$ cargo add serde ``` And then you can create the config struct, ```rust -extern crate serde; -#[macro_use] -extern crate serde_derive; +use serde::{Serialize, Deserialize}; -... +fn main() { #[derive(Debug, Default, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] pub struct WordcountConfig { pub ignores: Vec, } +} ``` Now we just need to deserialize the `WordcountConfig` from our `RenderContext` diff --git a/src/book/mod.rs b/src/book/mod.rs index f581b26701..853e8ed083 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -310,7 +310,7 @@ impl MDBook { let mut extern_args = ExternArgs::new(); if let Some(manifest) = &self.config.rust.manifest { - extern_args.load(&manifest)?; + extern_args.load(&self.root.join(manifest))?; } let mut failed = false; From d62904bb73313a6a3de5f4bbad7feeb5d01f2cab Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 16 Dec 2024 21:47:12 -0500 Subject: [PATCH 19/26] revert pulldown-cmark from 0.12.2 to 0.10.0 (sigh) --- Cargo.lock | 25 ++++++++++++++++++------- Cargo.toml | 4 ++-- src/renderer/html_handlebars/search.rs | 1 - 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 243d3bfd6d..08e58b1dac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1173,7 +1173,7 @@ dependencies = [ "pathdiff", "predicates", "pretty_assertions", - "pulldown-cmark", + "pulldown-cmark 0.10.3", "regex", "select", "semver", @@ -1195,7 +1195,7 @@ dependencies = [ "anyhow", "clap", "mdbook", - "pulldown-cmark", + "pulldown-cmark 0.10.3", "pulldown-cmark-to-cmark", "semver", "serde", @@ -1207,7 +1207,7 @@ name = "mdbook-remove-emphasis" version = "0.1.0" dependencies = [ "mdbook", - "pulldown-cmark", + "pulldown-cmark 0.12.2", "pulldown-cmark-to-cmark", "serde_json", ] @@ -1627,6 +1627,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" +dependencies = [ + "bitflags 2.6.0", + "memchr", + "pulldown-cmark-escape", + "unicase", +] + [[package]] name = "pulldown-cmark" version = "0.12.2" @@ -1635,15 +1647,14 @@ checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ "bitflags 2.6.0", "memchr", - "pulldown-cmark-escape", "unicase", ] [[package]] name = "pulldown-cmark-escape" -version = "0.11.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" [[package]] name = "pulldown-cmark-to-cmark" @@ -1651,7 +1662,7 @@ version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e02b63adcb49f2eb675b1694b413b3e9fedbf549dfe2cc98727ad97a0c30650" dependencies = [ - "pulldown-cmark", + "pulldown-cmark 0.12.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ecdbb55806..f20e0008e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [".", "examples/remove-emphasis/mdbook-remove-emphasis", "guide"] anyhow = "1.0.71" clap = { version = "4.3.12", features = ["cargo", "wrap_help"] } mdbook = { path = "." } -pulldown-cmark = { version = "0.12.2", default-features = false, features = [ +pulldown-cmark = { version = "0.10.0", default-features = false, features = [ "html", ] } # Do not update, part of the public api. serde = { version = "1.0.163", features = ["derive"] } @@ -49,7 +49,7 @@ serde.workspace = true serde_json.workspace = true shlex = "1.3.0" tempfile = "3.4.0" -toml = "0.5.11" # Do not update, see https://github.com/rust-lang/mdBook/issues/2037 +toml = "0.5.11" # Do not update, see https://github.com/rust-lang/mdBook/issues/2037 topological-sort = "0.2.2" # Watch feature diff --git a/src/renderer/html_handlebars/search.rs b/src/renderer/html_handlebars/search.rs index 31ce20e337..c03eb4f867 100644 --- a/src/renderer/html_handlebars/search.rs +++ b/src/renderer/html_handlebars/search.rs @@ -206,7 +206,6 @@ fn render_item( body.push_str(&format!(" [{number}] ")); } Event::TaskListMarker(_checked) => {} - Event::InlineMath(_) | Event::DisplayMath(_) => {} } } From b2eb96c2868d4f3ea755d9d12f2224d0a770611f Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 16 Dec 2024 22:23:03 -0500 Subject: [PATCH 20/26] finalize doc updates for new feature --- CHANGELOG.md | 6 ++++++ guide/src/cli/test.md | 5 +++-- guide/src/format/mdbook.md | 11 +++++++---- guide/src/guide/README.md | 3 +-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 345cbedb16..2ddf8022dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## mdBook 0.4.43+ (tba) + +### Fixed + +- Allow doctests to use external crates by referencing a `Cargo.toml` + ## mdBook 0.4.43 [v0.4.42...v0.4.43](https://github.com/rust-lang/mdBook/compare/v0.4.42...v0.4.43) diff --git a/guide/src/cli/test.md b/guide/src/cli/test.md index 4549fe4d5e..e8fd9210e6 100644 --- a/guide/src/cli/test.md +++ b/guide/src/cli/test.md @@ -7,7 +7,9 @@ For example, of code samples that could become outdated as the language evolves. MdBook supports a `test` command which runs code samples in your book as doc tests to verify they -will compile, and, optionally, run correctly. +will compile, and, optionally, run correctly. +For details on how to specify the test to be done and outcome to be expected, see [Code Blocks](/format/mdbook.md#code-blocks). + At the moment, mdBook only supports doc *tests* written in Rust, although code samples can be written and *displayed* in many programming languages. #### Specify a directory @@ -36,7 +38,6 @@ book using the chapter name or the relative path to the chapter. ***Note*** This argument is deprecated. Since Rust edition 2018, the compiler needs an explicit `--extern` argument for each external crate used in a doc test, it no longer simply scans the library path for likely-looking crates. New projects should list external crates as dependencies in a **Cargo.toml** file and reference that file in your ***book.toml***, as described in [rust configuration](/format/configuration/general.html#rust-options). - The `--library-path` (`-L`) option allows you to add directories to the library search path used by `rustdoc` when it builds and tests the examples. Multiple directories can be specified with multiple options (`-L foo -L bar`) or with a diff --git a/guide/src/format/mdbook.md b/guide/src/format/mdbook.md index 15974bcfb0..b2d73a5d48 100644 --- a/guide/src/format/mdbook.md +++ b/guide/src/format/mdbook.md @@ -1,8 +1,8 @@ # mdBook-specific features -# Features for code blocks +# Code blocks -These capabilities primarily affect how the user sees or interacts with code samples in your book and are supported directly by mdBook. Some also affect documentation tests (for which mdBook invokes `rustdoc --test`): this is detailed in the sections below. +These capabilities primarily affect how the user sees or interacts with code samples in your book and are supported directly by mdBook. Some also affect running the sample as a documentation test. (for which mdBook invokes `rustdoc --test`), so : this is detailed in the sections below. ## Hiding code lines @@ -12,7 +12,7 @@ For the Rust language, you can use the `#` character as a prefix which will hide [rustdoc-hide]: https://doc.rust-lang.org/stable/rustdoc/write-documentation/documentation-tests.html#hiding-portions-of-the-example -```bash +```text # fn main() { let x = 5; let y = 6; @@ -44,7 +44,7 @@ python = "~" The prefix will hide any lines that begin with the given prefix. With the python prefix shown above, this: -```bash +```text ~hidden() nothidden(): ~ hidden() @@ -155,6 +155,7 @@ interpreting them. ```` ## Including portions of a file + Often you only need a specific part of the file, e.g. relevant lines for an example. We support four different modes of partial includes: @@ -179,6 +180,7 @@ the regex `ANCHOR_END:\s*[\w_-]+`. This allows you to put anchors in any kind of commented line. Consider the following file to include: + ```rs /* ANCHOR: all */ @@ -196,6 +198,7 @@ impl System for MySystem { ... } ``` Then in the book, all you have to do is: + ````hbs Here is a component: ```rust,no_run,noplayground diff --git a/guide/src/guide/README.md b/guide/src/guide/README.md index 31fed08baf..d0061cbcbd 100644 --- a/guide/src/guide/README.md +++ b/guide/src/guide/README.md @@ -4,5 +4,4 @@ This user guide provides an introduction to basic concepts of using mdBook. - [Installation](installation.md) - [Reading Books](reading.md) -- [Creating a Book](creating.md) -- [Writing Code Samples](writing.md) +- [Creating a Book](creating.md)q From 248f490d8cf8a1df8ae787e6569c0975c8315250 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Mon, 16 Dec 2024 23:24:54 -0500 Subject: [PATCH 21/26] All doctests in mdbook project now pass. --- guide/src/for_developers/preprocessors.md | 7 ++++++- guide/src/format/mathjax.md | 9 +++++---- guide/src/format/mdbook.md | 6 +++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/guide/src/for_developers/preprocessors.md b/guide/src/for_developers/preprocessors.md index 3cd88797fb..7a52fce25d 100644 --- a/guide/src/for_developers/preprocessors.md +++ b/guide/src/for_developers/preprocessors.md @@ -66,7 +66,12 @@ translate events back into markdown text. The following code block shows how to remove all emphasis from markdown, without accidentally breaking the document. -```rust,no_run +```rust,compile_fail +# // tagged compile_fail because +# // sample fails to compile here: +# // "trait Borrow not implemented for pulldown_cmark_to_cmark::..." +# // Probably due to version skew on pulldown-cmark +# // between examples/remove-emphasis/Cargo.toml and /Cargo.toml {{#rustdoc_include ../../../examples/remove-emphasis/mdbook-remove-emphasis/src/main.rs:remove_emphasis}} ``` diff --git a/guide/src/format/mathjax.md b/guide/src/format/mathjax.md index 3dd792159d..642a3aaa19 100644 --- a/guide/src/format/mathjax.md +++ b/guide/src/format/mathjax.md @@ -20,24 +20,25 @@ extra backslash to work. Hopefully this limitation will be lifted soon. > to add _two extra_ backslashes (e.g., `\begin{cases} \frac 1 2 \\\\ \frac 3 4 > \end{cases}`). - ### Inline equations + Inline equations are delimited by `\\(` and `\\)`. So for example, to render the following inline equation \\( \int x dx = \frac{x^2}{2} + C \\) you would write the following: -``` + +```text \\( \int x dx = \frac{x^2}{2} + C \\) ``` ### Block equations + Block equations are delimited by `\\[` and `\\]`. To render the following equation \\[ \mu = \frac{1}{N} \sum_{i=0} x_i \\] - you would write: -```bash +```text \\[ \mu = \frac{1}{N} \sum_{i=0} x_i \\] ``` diff --git a/guide/src/format/mdbook.md b/guide/src/format/mdbook.md index b2d73a5d48..f3bada29c3 100644 --- a/guide/src/format/mdbook.md +++ b/guide/src/format/mdbook.md @@ -230,7 +230,7 @@ Rustdoc will use the complete example when you run `mdbook test`. For example, consider a file named `file.rs` that contains this Rust program: -```rust +```rust,editable fn main() { let x = add_one(2); assert_eq!(x, 3); @@ -320,8 +320,8 @@ To configure this: 1. Create a ***Cargo.toml*** file with a `[package.dependencies]` section that defines a dependency for each `` you want to use in any sample. If your book is already embedded in an existing Cargo project, you may be able to use the existing project `Cargo.toml`. 2. In your ***book.toml***: - * configure the path to ***Cargo.toml** in `rust.manifest`, as described in [rust configuration](/format/configuration/general.html#rust-options). - * remove `rust.edition` if it is configured. The edition of rust compiler will be as specified in the ***Cargo.toml***. + * configure the path to ***Cargo.toml*** in `rust.manifest`, as described in [rust configuration](/format/configuration/general.html#rust-options). + * remove `rust.edition` if it is configured. The default rust edition will be as specified in the ***Cargo.toml*** (though this can be overridden for a specific code block). * Refrain from invoking `mdbook test` with `-L` or `--library-path` argument. This, too, will be inferred from cargo project configuration # Features for general content From 38c8f5bfee00502641b984116a053d5b0a537f87 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Wed, 18 Dec 2024 16:15:56 -0500 Subject: [PATCH 22/26] Provide error context if can't find or open Cargo.toml; Prefer edition setting from Cargo.toml over book.toml; sadly, Cargo also overrides command line. --- src/book/mod.rs | 7 ++++++- src/utils/extern_args.rs | 23 +++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/book/mod.rs b/src/book/mod.rs index 853e8ed083..e8825f16ee 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -344,7 +344,11 @@ impl MDBook { .args(&library_args) // also need --extern for doctest to actually work .args(extern_args.get_args()); - if let Some(edition) = self.config.rust.edition { + // rustdoc edition from cargo manifest takes precedence over book.toml + // bugbug but also takes precedence over command line flag -- that seems rude. + if extern_args.edition != "" { + cmd.args(["--edition", &extern_args.edition]); + } else if let Some(edition) = self.config.rust.edition { match edition { RustEdition::E2015 => { cmd.args(["--edition", "2015"]); @@ -361,6 +365,7 @@ impl MDBook { } } + // bugbug Why show color in hidden invocation of rustdoc? if color_output { cmd.args(["--color", "always"]); } diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index a5eb88c578..b2e88ddd3c 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -42,9 +42,13 @@ use std::process::Command; #[derive(Debug)] pub struct ExternArgs { - edition: String, // where default value of "" means arg wasn't specified - crate_name: String, + /// rust edition as specified in manifest + pub edition: String, // where default value of "" means arg wasn't specified + /// crate name as specified in manifest + pub crate_name: String, + // accumulated library path(s), as observed from live cargo run lib_list: Vec, + // explicit extern crates, as observed from live cargo run extern_list: Vec, } @@ -65,11 +69,18 @@ impl ExternArgs { pub fn load(&mut self, cargo_path: &Path) -> Result<&Self> { // find Cargo.toml and determine the package name and lib or bin source file. let proj_root = cargo_path - .canonicalize()? + .canonicalize() + .context(format!( + "can't find cargo manifest {}", + &cargo_path.to_string_lossy() + ))? .parent() .ok_or(anyhow!("can't find parent of {:?}", cargo_path))? .to_owned(); - let mut manifest = Manifest::from_path(&cargo_path)?; + let mut manifest = Manifest::from_path(&cargo_path).context(format!( + "can't open cargo manifest {}", + &cargo_path.to_string_lossy() + ))?; manifest.complete_from_path(&proj_root)?; // try real hard to determine bin or lib let package = manifest .package @@ -204,10 +215,6 @@ impl ExternArgs { /// provide the parsed external args used to invoke rustdoc (--edition, -L and --extern). pub fn get_args(&self) -> Vec { let mut ret_val: Vec = vec![]; - if self.edition != "" { - ret_val.push("--edition".to_owned()); - ret_val.push(self.edition.clone()); - }; for i in &self.lib_list { ret_val.push("-L".to_owned()); ret_val.push(i.clone()); From ab5b3ab0328a42dd7763548ec271bd87707b5579 Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Thu, 2 Jan 2025 12:20:19 -0500 Subject: [PATCH 23/26] Fix unintended change, per review --- guide/src/guide/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/guide/README.md b/guide/src/guide/README.md index d0061cbcbd..90deb10e74 100644 --- a/guide/src/guide/README.md +++ b/guide/src/guide/README.md @@ -4,4 +4,4 @@ This user guide provides an introduction to basic concepts of using mdBook. - [Installation](installation.md) - [Reading Books](reading.md) -- [Creating a Book](creating.md)q +- [Creating a Book](creating.md) From 730e11c694c09d8f13171327a41f4fa2ead3805e Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Thu, 2 Jan 2025 12:48:14 -0500 Subject: [PATCH 24/26] fix clippy nags in files I changed --- src/book/mod.rs | 2 +- src/utils/extern_args.rs | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/book/mod.rs b/src/book/mod.rs index e8825f16ee..ff99a016fb 100644 --- a/src/book/mod.rs +++ b/src/book/mod.rs @@ -346,7 +346,7 @@ impl MDBook { // rustdoc edition from cargo manifest takes precedence over book.toml // bugbug but also takes precedence over command line flag -- that seems rude. - if extern_args.edition != "" { + if !extern_args.edition.is_empty() { cmd.args(["--edition", &extern_args.edition]); } else if let Some(edition) = self.config.rust.edition { match edition { diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index b2e88ddd3c..db07aec161 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -65,7 +65,6 @@ impl ExternArgs { /// Run a `cargo build` to see what args Cargo is using for library paths and extern crates. /// Touch a source file in the crate to ensure something is compiled and the args will be visible. - pub fn load(&mut self, cargo_path: &Path) -> Result<&Self> { // find Cargo.toml and determine the package name and lib or bin source file. let proj_root = cargo_path @@ -77,7 +76,7 @@ impl ExternArgs { .parent() .ok_or(anyhow!("can't find parent of {:?}", cargo_path))? .to_owned(); - let mut manifest = Manifest::from_path(&cargo_path).context(format!( + let mut manifest = Manifest::from_path(cargo_path).context(format!( "can't open cargo manifest {}", &cargo_path.to_string_lossy() ))?; @@ -107,7 +106,7 @@ impl ExternArgs { let try_path: PathBuf = proj_root.join("src").join(fname); if try_path.exists() { touch(&try_path)?; - self.run_cargo(&proj_root, &cargo_path)?; + self.run_cargo(&proj_root, cargo_path)?; return Ok(self); // file should be closed when f goes out of scope at bottom of this loop } @@ -117,7 +116,7 @@ impl ExternArgs { fn run_cargo(&mut self, proj_root: &Path, manifest_path: &Path) -> Result<&Self> { let mut cmd = Command::new("cargo"); - cmd.current_dir(&proj_root) + cmd.current_dir(proj_root) .arg("build") .arg("--verbose") .arg("--manifest-path") @@ -138,7 +137,7 @@ impl ExternArgs { //ultimatedebug std::fs::write(proj_root.join("mdbook_cargo_out.txt"), &output.stderr)?; let cmd_resp: &str = std::str::from_utf8(&output.stderr)?; - self.parse_response(&self.crate_name.clone(), &cmd_resp)?; + self.parse_response(self.crate_name.clone().as_str(), cmd_resp)?; Ok(self) } @@ -148,7 +147,7 @@ impl ExternArgs { /// The response may contain multiple builds, scan for the one that corresponds to the doctest crate. /// /// > This parser is broken, doesn't handle arg values with embedded spaces (single quoted). - /// Fortunately, the args we care about (so far) don't have those kinds of values. + /// > Fortunately, the args we care about (so far) don't have those kinds of values. pub fn parse_response(&mut self, my_crate: &str, buf: &str) -> Result<()> { let mut builds_ignored = 0; @@ -200,7 +199,7 @@ impl ExternArgs { }; } - if self.extern_list.len() == 0 || self.lib_list.len() == 0 { + if self.extern_list.is_empty() || self.lib_list.is_empty() { bail!("Couldn't extract -L or --extern args from Cargo, is current directory == cargo project root?"); } @@ -227,6 +226,12 @@ impl ExternArgs { } } +impl Default for ExternArgs { + fn default() -> Self { + Self::new() + } +} + fn my_display_edition(edition: Edition) -> String { match edition { Edition::E2015 => "2015", From c209369218311cb3c3912c9f8714f5bffb06506d Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Thu, 2 Jan 2025 13:01:53 -0500 Subject: [PATCH 25/26] fix unit test failure --- src/utils/extern_args.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/extern_args.rs b/src/utils/extern_args.rs index db07aec161..9520eb931f 100644 --- a/src/utils/extern_args.rs +++ b/src/utils/extern_args.rs @@ -292,7 +292,11 @@ mod test { ea.parse_response("leptos_book", &test_str)?; let args = ea.get_args(); - assert_eq!(20, args.len()); + + assert_eq!(ea.edition, "2021"); + assert_eq!(ea.crate_name, "leptos_book"); + + assert_eq!(18, args.len()); assert_eq!(1, args.iter().filter(|i| *i == "-L").count()); assert_eq!(8, args.iter().filter(|i| *i == "--extern").count()); From b9be9bcb79ddc4abe6db142dd0ec15e0a4c5f88d Mon Sep 17 00:00:00 2001 From: Bob Hyman Date: Sat, 8 Nov 2025 16:58:01 -0500 Subject: [PATCH 26/26] Migrate changes for ExternArgs from root to crates mdbook-core and mdbook-driver. --- Cargo.lock | 73 +- Cargo.toml | 1 + crates/mdbook-core/Cargo.toml | 1 + crates/mdbook-core/src/utils/extern_args.rs | 17 +- crates/mdbook-core/src/utils/mod.rs | 1 + crates/mdbook-driver/src/mdbook.rs | 12 +- src/book/mod.rs | 911 -------------------- src/utils/mod.rs | 545 ------------ tests/testsuite/config.rs | 2 +- 9 files changed, 90 insertions(+), 1473 deletions(-) delete mode 100644 src/book/mod.rs delete mode 100644 src/utils/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 23a2a10d6f..33e1a01f74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -209,6 +209,17 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cargo-manifest" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d8af896b707212cd0e99c112a78c9497dd32994192a463ed2f7419d29bd8c6" +dependencies = [ + "serde", + "thiserror", + "toml 0.8.23", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -603,7 +614,7 @@ dependencies = [ "mdbook-preprocessor", "semver", "serde_json", - "toml", + "toml 0.9.8", ] [[package]] @@ -983,7 +994,7 @@ dependencies = [ "snapbox", "tempfile", "tokio", - "toml", + "toml 0.9.8", "tower-http", "tracing", "tracing-subscriber", @@ -999,11 +1010,12 @@ name = "mdbook-core" version = "0.5.0-beta.2" dependencies = [ "anyhow", + "cargo-manifest", "regex", "serde", "serde_json", "tempfile", - "toml", + "toml 0.9.8", "tracing", ] @@ -1024,7 +1036,7 @@ dependencies = [ "serde_json", "shlex", "tempfile", - "toml", + "toml 0.9.8", "topological-sort", "tracing", ] @@ -1050,7 +1062,7 @@ dependencies = [ "serde_json", "sha2", "tempfile", - "toml", + "toml 0.9.8", "tracing", ] @@ -1677,6 +1689,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.0.3" @@ -1979,6 +2000,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit", +] + [[package]] name = "toml" version = "0.9.8" @@ -1987,13 +2021,22 @@ checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ "indexmap", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.0.3", + "toml_datetime 0.7.3", "toml_parser", "toml_writer", "winnow", ] +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.7.3" @@ -2003,6 +2046,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "winnow", +] + [[package]] name = "toml_parser" version = "1.0.4" @@ -2357,6 +2413,9 @@ name = "winnow" version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen" diff --git a/Cargo.toml b/Cargo.toml index 5d99fa6c1e..af2e77b8cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ rust-version = "1.88.0" # Keep in sync with installation.md and .github/workflow [workspace.dependencies] anyhow = "1.0.100" axum = "0.8.6" +cargo-manifest = "0.19.1" clap = { version = "4.5.51", features = ["cargo", "wrap_help"] } clap_complete = "4.5.60" ego-tree = "0.10.0" diff --git a/crates/mdbook-core/Cargo.toml b/crates/mdbook-core/Cargo.toml index f331fb06f9..c8283d77b5 100644 --- a/crates/mdbook-core/Cargo.toml +++ b/crates/mdbook-core/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true +cargo-manifest.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/mdbook-core/src/utils/extern_args.rs b/crates/mdbook-core/src/utils/extern_args.rs index 9520eb931f..bc95fd12aa 100644 --- a/crates/mdbook-core/src/utils/extern_args.rs +++ b/crates/mdbook-core/src/utils/extern_args.rs @@ -1,14 +1,14 @@ //! Get "compiler" args from cargo use crate::errors::*; -use anyhow::anyhow; +use anyhow::{Context, anyhow, bail}; use cargo_manifest::{Edition, Manifest, MaybeInherited::Local}; -use log::{debug, info}; use std::fs; use std::fs::File; use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::process::Command; +use tracing::{debug, info}; /// Get the arguments needed to invoke rustc so it can find external crates /// when invoked by rustdoc to compile doctests. @@ -23,8 +23,8 @@ use std::process::Command; /// Example: /// ```rust /// -/// use mdbook::utils::extern_args::ExternArgs; -/// # use mdbook::errors::*; +/// use mdbook_core::utils::extern_args::ExternArgs; +/// # use mdbook_core::errors::*; /// /// # fn main() -> Result<()> { /// // Get cargo to say what the compiler args need to be... @@ -86,7 +86,7 @@ impl ExternArgs { .expect("doctest Cargo.toml must include a [package] section"); self.crate_name = package.name.replace('-', "_"); // maybe cargo shouldn't allow packages to include non-identifier characters? - // in any case, this won't work when default crate doesn't have package name (which I'm sure cargo allows somehow or another) + // in any case, this won't work when default crate doesn't have package name (which I'm sure cargo allows somehow or another) self.edition = if let Some(Local(edition)) = package.edition { my_display_edition(edition) } else { @@ -174,7 +174,8 @@ impl ExternArgs { if dep_arg.ends_with(".rmeta") { debug!( "Build referenced {}, converted to .rlib hoping that actual file will be there in time.", - dep_arg); + dep_arg + ); dep_arg = dep_arg.replace(".rmeta", ".rlib"); } self.extern_list.push(dep_arg); @@ -200,7 +201,9 @@ impl ExternArgs { } if self.extern_list.is_empty() || self.lib_list.is_empty() { - bail!("Couldn't extract -L or --extern args from Cargo, is current directory == cargo project root?"); + bail!( + "Couldn't extract -L or --extern args from Cargo, is current directory == cargo project root?" + ); } debug!( diff --git a/crates/mdbook-core/src/utils/mod.rs b/crates/mdbook-core/src/utils/mod.rs index 01bd2a6d82..146fa320aa 100644 --- a/crates/mdbook-core/src/utils/mod.rs +++ b/crates/mdbook-core/src/utils/mod.rs @@ -4,6 +4,7 @@ use anyhow::Error; use std::fmt::Write; use tracing::error; +pub mod extern_args; pub mod fs; mod html; mod toml_ext; diff --git a/crates/mdbook-driver/src/mdbook.rs b/crates/mdbook-driver/src/mdbook.rs index 80ae692b92..3fb7747904 100644 --- a/crates/mdbook-driver/src/mdbook.rs +++ b/crates/mdbook-driver/src/mdbook.rs @@ -8,7 +8,7 @@ use anyhow::{Context, Error, Result, bail}; use indexmap::IndexMap; use mdbook_core::book::{Book, BookItem, BookItems}; use mdbook_core::config::{Config, RustEdition}; -use mdbook_core::utils::fs; +use mdbook_core::utils::{extern_args, fs}; use mdbook_html::HtmlHandlebars; use mdbook_preprocessor::{Preprocessor, PreprocessorContext}; use mdbook_renderer::{RenderContext, Renderer}; @@ -271,6 +271,13 @@ impl MDBook { let (book, _) = self.preprocess_book(&TestRenderer)?; let color_output = std::io::stderr().is_terminal(); + // get extra args we'll need for rustdoc, if config points to a cargo project + + let mut extern_args = extern_args::ExternArgs::new(); + if let Some(manifest) = &self.config.rust.manifest { + extern_args.load(&self.root.join(manifest))?; + } + let mut failed = false; for item in book.iter() { if let BookItem::Chapter(ref ch) = *item { @@ -298,7 +305,8 @@ impl MDBook { cmd.current_dir(temp_dir.path()) .arg(chapter_path) .arg("--test") - .args(&library_args); + .args(&library_args) + .args(extern_args.get_args()); if let Some(edition) = self.config.rust.edition { match edition { diff --git a/src/book/mod.rs b/src/book/mod.rs deleted file mode 100644 index ff99a016fb..0000000000 --- a/src/book/mod.rs +++ /dev/null @@ -1,911 +0,0 @@ -//! The internal representation of a book and infrastructure for loading it from -//! disk and building it. -//! -//! For examples on using `MDBook`, consult the [top-level documentation][1]. -//! -//! [1]: ../index.html - -#[allow(clippy::module_inception)] -mod book; -mod init; -mod summary; - -pub use self::book::{load_book, Book, BookItem, BookItems, Chapter}; -pub use self::init::BookBuilder; -pub use self::summary::{parse_summary, Link, SectionNumber, Summary, SummaryItem}; - -use log::{debug, error, info, log_enabled, trace, warn}; -use std::ffi::OsString; -use std::io::{IsTerminal, Write}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use tempfile::Builder as TempFileBuilder; -use toml::Value; -use topological_sort::TopologicalSort; - -use crate::errors::*; -use crate::preprocess::{ - CmdPreprocessor, IndexPreprocessor, LinkPreprocessor, Preprocessor, PreprocessorContext, -}; -use crate::renderer::{CmdRenderer, HtmlHandlebars, MarkdownRenderer, RenderContext, Renderer}; -use crate::utils; - -use crate::config::{Config, RustEdition}; -use crate::utils::extern_args::ExternArgs; - -/// The object used to manage and build a book. -pub struct MDBook { - /// The book's root directory. - pub root: PathBuf, - /// The configuration used to tweak now a book is built. - pub config: Config, - /// A representation of the book's contents in memory. - pub book: Book, - renderers: Vec>, - - /// List of pre-processors to be run on the book. - preprocessors: Vec>, -} - -impl MDBook { - /// Load a book from its root directory on disk. - pub fn load>(book_root: P) -> Result { - let book_root = book_root.into(); - let config_location = book_root.join("book.toml"); - - // the book.json file is no longer used, so we should emit a warning to - // let people know to migrate to book.toml - if book_root.join("book.json").exists() { - warn!("It appears you are still using book.json for configuration."); - warn!("This format is no longer used, so you should migrate to the"); - warn!("book.toml format."); - warn!("Check the user guide for migration information:"); - warn!("\thttps://rust-lang.github.io/mdBook/format/config.html"); - } - - let mut config = if config_location.exists() { - debug!("Loading config from {}", config_location.display()); - Config::from_disk(&config_location)? - } else { - Config::default() - }; - - config.update_from_env(); - - if let Some(html_config) = config.html_config() { - if html_config.google_analytics.is_some() { - warn!( - "The output.html.google-analytics field has been deprecated; \ - it will be removed in a future release.\n\ - Consider placing the appropriate site tag code into the \ - theme/head.hbs file instead.\n\ - The tracking code may be found in the Google Analytics Admin page.\n\ - " - ); - } - if html_config.curly_quotes { - warn!( - "The output.html.curly-quotes field has been renamed to \ - output.html.smart-punctuation.\n\ - Use the new name in book.toml to remove this warning." - ); - } - } - - if log_enabled!(log::Level::Trace) { - for line in format!("Config: {config:#?}").lines() { - trace!("{}", line); - } - } - - MDBook::load_with_config(book_root, config) - } - - /// Load a book from its root directory using a custom `Config`. - pub fn load_with_config>(book_root: P, config: Config) -> Result { - let root = book_root.into(); - - let src_dir = root.join(&config.book.src); - let book = book::load_book(src_dir, &config.build)?; - - let renderers = determine_renderers(&config); - let preprocessors = determine_preprocessors(&config)?; - - Ok(MDBook { - root, - config, - book, - renderers, - preprocessors, - }) - } - - /// Load a book from its root directory using a custom `Config` and a custom summary. - pub fn load_with_config_and_summary>( - book_root: P, - config: Config, - summary: Summary, - ) -> Result { - let root = book_root.into(); - - let src_dir = root.join(&config.book.src); - let book = book::load_book_from_disk(&summary, src_dir)?; - - let renderers = determine_renderers(&config); - let preprocessors = determine_preprocessors(&config)?; - - Ok(MDBook { - root, - config, - book, - renderers, - preprocessors, - }) - } - - /// Returns a flat depth-first iterator over the elements of the book, - /// it returns a [`BookItem`] enum: - /// `(section: String, bookitem: &BookItem)` - /// - /// ```no_run - /// # use mdbook::MDBook; - /// # use mdbook::book::BookItem; - /// # let book = MDBook::load("mybook").unwrap(); - /// for item in book.iter() { - /// match *item { - /// BookItem::Chapter(ref chapter) => {}, - /// BookItem::Separator => {}, - /// BookItem::PartTitle(ref title) => {} - /// } - /// } - /// - /// // would print something like this: - /// // 1. Chapter 1 - /// // 1.1 Sub Chapter - /// // 1.2 Sub Chapter - /// // 2. Chapter 2 - /// // - /// // etc. - /// ``` - pub fn iter(&self) -> BookItems<'_> { - self.book.iter() - } - - /// `init()` gives you a `BookBuilder` which you can use to setup a new book - /// and its accompanying directory structure. - /// - /// The `BookBuilder` creates some boilerplate files and directories to get - /// you started with your book. - /// - /// ```text - /// book-test/ - /// ├── book - /// └── src - /// ├── chapter_1.md - /// └── SUMMARY.md - /// ``` - /// - /// It uses the path provided as the root directory for your book, then adds - /// in a `src/` directory containing a `SUMMARY.md` and `chapter_1.md` file - /// to get you started. - pub fn init>(book_root: P) -> BookBuilder { - BookBuilder::new(book_root) - } - - /// Tells the renderer to build our book and put it in the build directory. - pub fn build(&self) -> Result<()> { - info!("Book building has started"); - - for renderer in &self.renderers { - self.execute_build_process(&**renderer)?; - } - - Ok(()) - } - - /// Run preprocessors and return the final book. - pub fn preprocess_book(&self, renderer: &dyn Renderer) -> Result<(Book, PreprocessorContext)> { - let preprocess_ctx = PreprocessorContext::new( - self.root.clone(), - self.config.clone(), - renderer.name().to_string(), - ); - let mut preprocessed_book = self.book.clone(); - for preprocessor in &self.preprocessors { - if preprocessor_should_run(&**preprocessor, renderer, &self.config) { - debug!("Running the {} preprocessor.", preprocessor.name()); - preprocessed_book = preprocessor.run(&preprocess_ctx, preprocessed_book)?; - } - } - Ok((preprocessed_book, preprocess_ctx)) - } - - /// Run the entire build process for a particular [`Renderer`]. - pub fn execute_build_process(&self, renderer: &dyn Renderer) -> Result<()> { - let (preprocessed_book, preprocess_ctx) = self.preprocess_book(renderer)?; - - let name = renderer.name(); - let build_dir = self.build_dir_for(name); - - let mut render_context = RenderContext::new( - self.root.clone(), - preprocessed_book, - self.config.clone(), - build_dir, - ); - render_context - .chapter_titles - .extend(preprocess_ctx.chapter_titles.borrow_mut().drain()); - - info!("Running the {} backend", renderer.name()); - renderer - .render(&render_context) - .with_context(|| "Rendering failed") - } - - /// You can change the default renderer to another one by using this method. - /// The only requirement is that your renderer implement the [`Renderer`] - /// trait. - pub fn with_renderer(&mut self, renderer: R) -> &mut Self { - self.renderers.push(Box::new(renderer)); - self - } - - /// Register a [`Preprocessor`] to be used when rendering the book. - pub fn with_preprocessor(&mut self, preprocessor: P) -> &mut Self { - self.preprocessors.push(Box::new(preprocessor)); - self - } - - /// Run `rustdoc` tests on the book, linking against the provided libraries. - pub fn test(&mut self, library_paths: Vec<&str>) -> Result<()> { - // test_chapter with chapter:None will run all tests. - self.test_chapter(library_paths, None) - } - - /// Run `rustdoc` tests on a specific chapter of the book, linking against the provided libraries. - /// If `chapter` is `None`, all tests will be run. - pub fn test_chapter(&mut self, library_paths: Vec<&str>, chapter: Option<&str>) -> Result<()> { - let cwd = std::env::current_dir()?; - let library_args: Vec = library_paths - .into_iter() - .flat_map(|path| { - let path = Path::new(path); - let path = if path.is_relative() { - cwd.join(path).into_os_string() - } else { - path.to_path_buf().into_os_string() - }; - [OsString::from("-L"), path] - }) - .collect(); - - let temp_dir = TempFileBuilder::new().prefix("mdbook-").tempdir()?; - - let mut chapter_found = false; - - struct TestRenderer; - impl Renderer for TestRenderer { - // FIXME: Is "test" the proper renderer name to use here? - fn name(&self) -> &str { - "test" - } - - fn render(&self, _: &RenderContext) -> Result<()> { - Ok(()) - } - } - - // Index Preprocessor is disabled so that chapter paths - // continue to point to the actual markdown files. - self.preprocessors = determine_preprocessors(&self.config)? - .into_iter() - .filter(|pre| pre.name() != IndexPreprocessor::NAME) - .collect(); - let (book, _) = self.preprocess_book(&TestRenderer)?; - - let color_output = std::io::stderr().is_terminal(); - - // get extra args we'll need for rustdoc, if config points to a cargo project. - - let mut extern_args = ExternArgs::new(); - if let Some(manifest) = &self.config.rust.manifest { - extern_args.load(&self.root.join(manifest))?; - } - - let mut failed = false; - for item in book.iter() { - if let BookItem::Chapter(ref ch) = *item { - let chapter_path = match ch.path { - Some(ref path) if !path.as_os_str().is_empty() => path, - _ => continue, - }; - - if let Some(chapter) = chapter { - if ch.name != chapter && chapter_path.to_str() != Some(chapter) { - if chapter == "?" { - info!("Skipping chapter '{}'...", ch.name); - } - continue; - } - } - chapter_found = true; - info!("Testing chapter '{}': {:?}", ch.name, chapter_path); - - // write preprocessed file to tempdir - let path = temp_dir.path().join(chapter_path); - let mut tmpf = utils::fs::create_file(&path)?; - tmpf.write_all(ch.content.as_bytes())?; - - let mut cmd = Command::new("rustdoc"); - cmd.current_dir(temp_dir.path()) - .arg(chapter_path) - .arg("--test") - .args(&library_args) // also need --extern for doctest to actually work - .args(extern_args.get_args()); - - // rustdoc edition from cargo manifest takes precedence over book.toml - // bugbug but also takes precedence over command line flag -- that seems rude. - if !extern_args.edition.is_empty() { - cmd.args(["--edition", &extern_args.edition]); - } else if let Some(edition) = self.config.rust.edition { - match edition { - RustEdition::E2015 => { - cmd.args(["--edition", "2015"]); - } - RustEdition::E2018 => { - cmd.args(["--edition", "2018"]); - } - RustEdition::E2021 => { - cmd.args(["--edition", "2021"]); - } - RustEdition::E2024 => { - cmd.args(["--edition", "2024"]); - } - } - } - - // bugbug Why show color in hidden invocation of rustdoc? - if color_output { - cmd.args(["--color", "always"]); - } - - debug!("running {:?}", cmd); - let output = cmd.output()?; - - if !output.status.success() { - failed = true; - error!( - "rustdoc returned an error:\n\ - \n--- stdout\n{}\n--- stderr\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - } - } - if failed { - bail!("One or more tests failed"); - } - if let Some(chapter) = chapter { - if !chapter_found { - bail!("Chapter not found: {}", chapter); - } - } - Ok(()) - } - - /// The logic for determining where a backend should put its build - /// artefacts. - /// - /// If there is only 1 renderer, put it in the directory pointed to by the - /// `build.build_dir` key in [`Config`]. If there is more than one then the - /// renderer gets its own directory within the main build dir. - /// - /// i.e. If there were only one renderer (in this case, the HTML renderer): - /// - /// - build/ - /// - index.html - /// - ... - /// - /// Otherwise if there are multiple: - /// - /// - build/ - /// - epub/ - /// - my_awesome_book.epub - /// - html/ - /// - index.html - /// - ... - /// - latex/ - /// - my_awesome_book.tex - /// - pub fn build_dir_for(&self, backend_name: &str) -> PathBuf { - let build_dir = self.root.join(&self.config.build.build_dir); - - if self.renderers.len() <= 1 { - build_dir - } else { - build_dir.join(backend_name) - } - } - - /// Get the directory containing this book's source files. - pub fn source_dir(&self) -> PathBuf { - self.root.join(&self.config.book.src) - } - - /// Get the directory containing the theme resources for the book. - pub fn theme_dir(&self) -> PathBuf { - self.config - .html_config() - .unwrap_or_default() - .theme_dir(&self.root) - } -} - -/// Look at the `Config` and try to figure out what renderers to use. -fn determine_renderers(config: &Config) -> Vec> { - let mut renderers = Vec::new(); - - if let Some(output_table) = config.get("output").and_then(Value::as_table) { - renderers.extend(output_table.iter().map(|(key, table)| { - if key == "html" { - Box::new(HtmlHandlebars::new()) as Box - } else if key == "markdown" { - Box::new(MarkdownRenderer::new()) as Box - } else { - interpret_custom_renderer(key, table) - } - })); - } - - // if we couldn't find anything, add the HTML renderer as a default - if renderers.is_empty() { - renderers.push(Box::new(HtmlHandlebars::new())); - } - - renderers -} - -const DEFAULT_PREPROCESSORS: &[&str] = &["links", "index"]; - -fn is_default_preprocessor(pre: &dyn Preprocessor) -> bool { - let name = pre.name(); - name == LinkPreprocessor::NAME || name == IndexPreprocessor::NAME -} - -/// Look at the `MDBook` and try to figure out what preprocessors to run. -fn determine_preprocessors(config: &Config) -> Result>> { - // Collect the names of all preprocessors intended to be run, and the order - // in which they should be run. - let mut preprocessor_names = TopologicalSort::::new(); - - if config.build.use_default_preprocessors { - for name in DEFAULT_PREPROCESSORS { - preprocessor_names.insert(name.to_string()); - } - } - - if let Some(preprocessor_table) = config.get("preprocessor").and_then(Value::as_table) { - for (name, table) in preprocessor_table.iter() { - preprocessor_names.insert(name.to_string()); - - let exists = |name| { - (config.build.use_default_preprocessors && DEFAULT_PREPROCESSORS.contains(&name)) - || preprocessor_table.contains_key(name) - }; - - if let Some(before) = table.get("before") { - let before = before.as_array().ok_or_else(|| { - Error::msg(format!( - "Expected preprocessor.{name}.before to be an array" - )) - })?; - for after in before { - let after = after.as_str().ok_or_else(|| { - Error::msg(format!( - "Expected preprocessor.{name}.before to contain strings" - )) - })?; - - if !exists(after) { - // Only warn so that preprocessors can be toggled on and off (e.g. for - // troubleshooting) without having to worry about order too much. - warn!( - "preprocessor.{}.after contains \"{}\", which was not found", - name, after - ); - } else { - preprocessor_names.add_dependency(name, after); - } - } - } - - if let Some(after) = table.get("after") { - let after = after.as_array().ok_or_else(|| { - Error::msg(format!("Expected preprocessor.{name}.after to be an array")) - })?; - for before in after { - let before = before.as_str().ok_or_else(|| { - Error::msg(format!( - "Expected preprocessor.{name}.after to contain strings" - )) - })?; - - if !exists(before) { - // See equivalent warning above for rationale - warn!( - "preprocessor.{}.before contains \"{}\", which was not found", - name, before - ); - } else { - preprocessor_names.add_dependency(before, name); - } - } - } - } - } - - // Now that all links have been established, queue preprocessors in a suitable order - let mut preprocessors = Vec::with_capacity(preprocessor_names.len()); - // `pop_all()` returns an empty vector when no more items are not being depended upon - for mut names in std::iter::repeat_with(|| preprocessor_names.pop_all()) - .take_while(|names| !names.is_empty()) - { - // The `topological_sort` crate does not guarantee a stable order for ties, even across - // runs of the same program. Thus, we break ties manually by sorting. - // Careful: `str`'s default sorting, which we are implicitly invoking here, uses code point - // values ([1]), which may not be an alphabetical sort. - // As mentioned in [1], doing so depends on locale, which is not desirable for deciding - // preprocessor execution order. - // [1]: https://doc.rust-lang.org/stable/std/cmp/trait.Ord.html#impl-Ord-14 - names.sort(); - for name in names { - let preprocessor: Box = match name.as_str() { - "links" => Box::new(LinkPreprocessor::new()), - "index" => Box::new(IndexPreprocessor::new()), - _ => { - // The only way to request a custom preprocessor is through the `preprocessor` - // table, so it must exist, be a table, and contain the key. - let table = &config.get("preprocessor").unwrap().as_table().unwrap()[&name]; - let command = get_custom_preprocessor_cmd(&name, table); - Box::new(CmdPreprocessor::new(name, command)) - } - }; - preprocessors.push(preprocessor); - } - } - - // "If `pop_all` returns an empty vector and `len` is not 0, there are cyclic dependencies." - // Normally, `len() == 0` is equivalent to `is_empty()`, so we'll use that. - if preprocessor_names.is_empty() { - Ok(preprocessors) - } else { - Err(Error::msg("Cyclic dependency detected in preprocessors")) - } -} - -fn get_custom_preprocessor_cmd(key: &str, table: &Value) -> String { - table - .get("command") - .and_then(Value::as_str) - .map(ToString::to_string) - .unwrap_or_else(|| format!("mdbook-{key}")) -} - -fn interpret_custom_renderer(key: &str, table: &Value) -> Box { - // look for the `command` field, falling back to using the key - // prepended by "mdbook-" - let table_dot_command = table - .get("command") - .and_then(Value::as_str) - .map(ToString::to_string); - - let command = table_dot_command.unwrap_or_else(|| format!("mdbook-{key}")); - - Box::new(CmdRenderer::new(key.to_string(), command)) -} - -/// Check whether we should run a particular `Preprocessor` in combination -/// with the renderer, falling back to `Preprocessor::supports_renderer()` -/// method if the user doesn't say anything. -/// -/// The `build.use-default-preprocessors` config option can be used to ensure -/// default preprocessors always run if they support the renderer. -fn preprocessor_should_run( - preprocessor: &dyn Preprocessor, - renderer: &dyn Renderer, - cfg: &Config, -) -> bool { - // default preprocessors should be run by default (if supported) - if cfg.build.use_default_preprocessors && is_default_preprocessor(preprocessor) { - return preprocessor.supports_renderer(renderer.name()); - } - - let key = format!("preprocessor.{}.renderers", preprocessor.name()); - let renderer_name = renderer.name(); - - if let Some(Value::Array(ref explicit_renderers)) = cfg.get(&key) { - return explicit_renderers - .iter() - .filter_map(Value::as_str) - .any(|name| name == renderer_name); - } - - preprocessor.supports_renderer(renderer_name) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::str::FromStr; - use toml::value::Table; - - #[test] - fn config_defaults_to_html_renderer_if_empty() { - let cfg = Config::default(); - - // make sure we haven't got anything in the `output` table - assert!(cfg.get("output").is_none()); - - let got = determine_renderers(&cfg); - - assert_eq!(got.len(), 1); - assert_eq!(got[0].name(), "html"); - } - - #[test] - fn add_a_random_renderer_to_the_config() { - let mut cfg = Config::default(); - cfg.set("output.random", Table::new()).unwrap(); - - let got = determine_renderers(&cfg); - - assert_eq!(got.len(), 1); - assert_eq!(got[0].name(), "random"); - } - - #[test] - fn add_a_random_renderer_with_custom_command_to_the_config() { - let mut cfg = Config::default(); - - let mut table = Table::new(); - table.insert("command".to_string(), Value::String("false".to_string())); - cfg.set("output.random", table).unwrap(); - - let got = determine_renderers(&cfg); - - assert_eq!(got.len(), 1); - assert_eq!(got[0].name(), "random"); - } - - #[test] - fn config_defaults_to_link_and_index_preprocessor_if_not_set() { - let cfg = Config::default(); - - // make sure we haven't got anything in the `preprocessor` table - assert!(cfg.get("preprocessor").is_none()); - - let got = determine_preprocessors(&cfg); - - assert!(got.is_ok()); - assert_eq!(got.as_ref().unwrap().len(), 2); - assert_eq!(got.as_ref().unwrap()[0].name(), "index"); - assert_eq!(got.as_ref().unwrap()[1].name(), "links"); - } - - #[test] - fn use_default_preprocessors_works() { - let mut cfg = Config::default(); - cfg.build.use_default_preprocessors = false; - - let got = determine_preprocessors(&cfg).unwrap(); - - assert_eq!(got.len(), 0); - } - - #[test] - fn can_determine_third_party_preprocessors() { - let cfg_str = r#" - [book] - title = "Some Book" - - [preprocessor.random] - - [build] - build-dir = "outputs" - create-missing = false - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - // make sure the `preprocessor.random` table exists - assert!(cfg.get_preprocessor("random").is_some()); - - let got = determine_preprocessors(&cfg).unwrap(); - - assert!(got.into_iter().any(|p| p.name() == "random")); - } - - #[test] - fn preprocessors_can_provide_their_own_commands() { - let cfg_str = r#" - [preprocessor.random] - command = "python random.py" - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - // make sure the `preprocessor.random` table exists - let random = cfg.get_preprocessor("random").unwrap(); - let random = get_custom_preprocessor_cmd("random", &Value::Table(random.clone())); - - assert_eq!(random, "python random.py"); - } - - #[test] - fn preprocessor_before_must_be_array() { - let cfg_str = r#" - [preprocessor.random] - before = 0 - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - assert!(determine_preprocessors(&cfg).is_err()); - } - - #[test] - fn preprocessor_after_must_be_array() { - let cfg_str = r#" - [preprocessor.random] - after = 0 - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - assert!(determine_preprocessors(&cfg).is_err()); - } - - #[test] - fn preprocessor_order_is_honored() { - let cfg_str = r#" - [preprocessor.random] - before = [ "last" ] - after = [ "index" ] - - [preprocessor.last] - after = [ "links", "index" ] - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - let preprocessors = determine_preprocessors(&cfg).unwrap(); - let index = |name| { - preprocessors - .iter() - .enumerate() - .find(|(_, preprocessor)| preprocessor.name() == name) - .unwrap() - .0 - }; - let assert_before = |before, after| { - if index(before) >= index(after) { - eprintln!("Preprocessor order:"); - for preprocessor in &preprocessors { - eprintln!(" {}", preprocessor.name()); - } - panic!("{before} should come before {after}"); - } - }; - - assert_before("index", "random"); - assert_before("index", "last"); - assert_before("random", "last"); - assert_before("links", "last"); - } - - #[test] - fn cyclic_dependencies_are_detected() { - let cfg_str = r#" - [preprocessor.links] - before = [ "index" ] - - [preprocessor.index] - before = [ "links" ] - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - assert!(determine_preprocessors(&cfg).is_err()); - } - - #[test] - fn dependencies_dont_register_undefined_preprocessors() { - let cfg_str = r#" - [preprocessor.links] - before = [ "random" ] - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - let preprocessors = determine_preprocessors(&cfg).unwrap(); - - assert!(!preprocessors - .iter() - .any(|preprocessor| preprocessor.name() == "random")); - } - - #[test] - fn dependencies_dont_register_builtin_preprocessors_if_disabled() { - let cfg_str = r#" - [preprocessor.random] - before = [ "links" ] - - [build] - use-default-preprocessors = false - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - let preprocessors = determine_preprocessors(&cfg).unwrap(); - - assert!(!preprocessors - .iter() - .any(|preprocessor| preprocessor.name() == "links")); - } - - #[test] - fn config_respects_preprocessor_selection() { - let cfg_str = r#" - [preprocessor.links] - renderers = ["html"] - "#; - - let cfg = Config::from_str(cfg_str).unwrap(); - - // double-check that we can access preprocessor.links.renderers[0] - let html = cfg - .get_preprocessor("links") - .and_then(|links| links.get("renderers")) - .and_then(Value::as_array) - .and_then(|renderers| renderers.get(0)) - .and_then(Value::as_str) - .unwrap(); - assert_eq!(html, "html"); - let html_renderer = HtmlHandlebars::default(); - let pre = LinkPreprocessor::new(); - - let should_run = preprocessor_should_run(&pre, &html_renderer, &cfg); - assert!(should_run); - } - - struct BoolPreprocessor(bool); - impl Preprocessor for BoolPreprocessor { - fn name(&self) -> &str { - "bool-preprocessor" - } - - fn run(&self, _ctx: &PreprocessorContext, _book: Book) -> Result { - unimplemented!() - } - - fn supports_renderer(&self, _renderer: &str) -> bool { - self.0 - } - } - - #[test] - fn preprocessor_should_run_falls_back_to_supports_renderer_method() { - let cfg = Config::default(); - let html = HtmlHandlebars::new(); - - let should_be = true; - let got = preprocessor_should_run(&BoolPreprocessor(should_be), &html, &cfg); - assert_eq!(got, should_be); - - let should_be = false; - let got = preprocessor_should_run(&BoolPreprocessor(should_be), &html, &cfg); - assert_eq!(got, should_be); - } -} diff --git a/src/utils/mod.rs b/src/utils/mod.rs deleted file mode 100644 index 4650339723..0000000000 --- a/src/utils/mod.rs +++ /dev/null @@ -1,545 +0,0 @@ -#![allow(missing_docs)] // FIXME: Document this - -pub mod extern_args; -pub mod fs; -mod string; -pub(crate) mod toml_ext; -use crate::errors::Error; -use log::error; -use once_cell::sync::Lazy; -use pulldown_cmark::{html, CodeBlockKind, CowStr, Event, Options, Parser, Tag, TagEnd}; -use regex::Regex; - -use std::borrow::Cow; -use std::collections::HashMap; -use std::fmt::Write; -use std::path::Path; - -pub use self::string::{ - take_anchored_lines, take_lines, take_rustdoc_include_anchored_lines, - take_rustdoc_include_lines, -}; - -/// Replaces multiple consecutive whitespace characters with a single space character. -pub fn collapse_whitespace(text: &str) -> Cow<'_, str> { - static RE: Lazy = Lazy::new(|| Regex::new(r"\s\s+").unwrap()); - RE.replace_all(text, " ") -} - -/// Convert the given string to a valid HTML element ID. -/// The only restriction is that the ID must not contain any ASCII whitespace. -pub fn normalize_id(content: &str) -> String { - content - .chars() - .filter_map(|ch| { - if ch.is_alphanumeric() || ch == '_' || ch == '-' { - Some(ch.to_ascii_lowercase()) - } else if ch.is_whitespace() { - Some('-') - } else { - None - } - }) - .collect::() -} - -/// Generate an ID for use with anchors which is derived from a "normalised" -/// string. -// This function should be made private when the deprecation expires. -#[deprecated(since = "0.4.16", note = "use unique_id_from_content instead")] -pub fn id_from_content(content: &str) -> String { - let mut content = content.to_string(); - - // Skip any tags or html-encoded stuff - static HTML: Lazy = Lazy::new(|| Regex::new(r"(<.*?>)").unwrap()); - content = HTML.replace_all(&content, "").into(); - const REPL_SUB: &[&str] = &["<", ">", "&", "'", """]; - for sub in REPL_SUB { - content = content.replace(sub, ""); - } - - // Remove spaces and hashes indicating a header - let trimmed = content.trim().trim_start_matches('#').trim(); - normalize_id(trimmed) -} - -/// Generate an ID for use with anchors which is derived from a "normalised" -/// string. -/// -/// Each ID returned will be unique, if the same `id_counter` is provided on -/// each call. -pub fn unique_id_from_content(content: &str, id_counter: &mut HashMap) -> String { - let id = { - #[allow(deprecated)] - id_from_content(content) - }; - - // If we have headers with the same normalized id, append an incrementing counter - let id_count = id_counter.entry(id.clone()).or_insert(0); - let unique_id = match *id_count { - 0 => id, - id_count => format!("{id}-{id_count}"), - }; - *id_count += 1; - unique_id -} - -/// Fix links to the correct location. -/// -/// This adjusts links, such as turning `.md` extensions to `.html`. -/// -/// `path` is the path to the page being rendered relative to the root of the -/// book. This is used for the `print.html` page so that links on the print -/// page go to the original location. Normal page rendering sets `path` to -/// None. Ideally, print page links would link to anchors on the print page, -/// but that is very difficult. -fn adjust_links<'a>(event: Event<'a>, path: Option<&Path>) -> Event<'a> { - static SCHEME_LINK: Lazy = Lazy::new(|| Regex::new(r"^[a-z][a-z0-9+.-]*:").unwrap()); - static MD_LINK: Lazy = - Lazy::new(|| Regex::new(r"(?P.*)\.md(?P#.*)?").unwrap()); - - fn fix<'a>(dest: CowStr<'a>, path: Option<&Path>) -> CowStr<'a> { - if dest.starts_with('#') { - // Fragment-only link. - if let Some(path) = path { - let mut base = path.display().to_string(); - if base.ends_with(".md") { - base.replace_range(base.len() - 3.., ".html"); - } - return format!("{base}{dest}").into(); - } else { - return dest; - } - } - // Don't modify links with schemes like `https`. - if !SCHEME_LINK.is_match(&dest) { - // This is a relative link, adjust it as necessary. - let mut fixed_link = String::new(); - if let Some(path) = path { - let base = path - .parent() - .expect("path can't be empty") - .to_str() - .expect("utf-8 paths only"); - if !base.is_empty() { - write!(fixed_link, "{base}/").unwrap(); - } - } - - if let Some(caps) = MD_LINK.captures(&dest) { - fixed_link.push_str(&caps["link"]); - fixed_link.push_str(".html"); - if let Some(anchor) = caps.name("anchor") { - fixed_link.push_str(anchor.as_str()); - } - } else { - fixed_link.push_str(&dest); - }; - return CowStr::from(fixed_link); - } - dest - } - - fn fix_html<'a>(html: CowStr<'a>, path: Option<&Path>) -> CowStr<'a> { - // This is a terrible hack, but should be reasonably reliable. Nobody - // should ever parse a tag with a regex. However, there isn't anything - // in Rust that I know of that is suitable for handling partial html - // fragments like those generated by pulldown_cmark. - // - // There are dozens of HTML tags/attributes that contain paths, so - // feel free to add more tags if desired; these are the only ones I - // care about right now. - static HTML_LINK: Lazy = - Lazy::new(|| Regex::new(r#"(<(?:a|img) [^>]*?(?:src|href)=")([^"]+?)""#).unwrap()); - - HTML_LINK - .replace_all(&html, |caps: ®ex::Captures<'_>| { - let fixed = fix(caps[2].into(), path); - format!("{}{}\"", &caps[1], fixed) - }) - .into_owned() - .into() - } - - match event { - Event::Start(Tag::Link { - link_type, - dest_url, - title, - id, - }) => Event::Start(Tag::Link { - link_type, - dest_url: fix(dest_url, path), - title, - id, - }), - Event::Start(Tag::Image { - link_type, - dest_url, - title, - id, - }) => Event::Start(Tag::Image { - link_type, - dest_url: fix(dest_url, path), - title, - id, - }), - Event::Html(html) => Event::Html(fix_html(html, path)), - Event::InlineHtml(html) => Event::InlineHtml(fix_html(html, path)), - _ => event, - } -} - -/// Wrapper around the pulldown-cmark parser for rendering markdown to HTML. -pub fn render_markdown(text: &str, smart_punctuation: bool) -> String { - render_markdown_with_path(text, smart_punctuation, None) -} - -pub fn new_cmark_parser(text: &str, smart_punctuation: bool) -> Parser<'_> { - let mut opts = Options::empty(); - opts.insert(Options::ENABLE_TABLES); - opts.insert(Options::ENABLE_FOOTNOTES); - opts.insert(Options::ENABLE_STRIKETHROUGH); - opts.insert(Options::ENABLE_TASKLISTS); - opts.insert(Options::ENABLE_HEADING_ATTRIBUTES); - if smart_punctuation { - opts.insert(Options::ENABLE_SMART_PUNCTUATION); - } - Parser::new_ext(text, opts) -} - -pub fn render_markdown_with_path( - text: &str, - smart_punctuation: bool, - path: Option<&Path>, -) -> String { - let mut s = String::with_capacity(text.len() * 3 / 2); - let p = new_cmark_parser(text, smart_punctuation); - let events = p - .map(clean_codeblock_headers) - .map(|event| adjust_links(event, path)) - .flat_map(|event| { - let (a, b) = wrap_tables(event); - a.into_iter().chain(b) - }); - - html::push_html(&mut s, events); - s -} - -/// Wraps tables in a `.table-wrapper` class to apply overflow-x rules to. -fn wrap_tables(event: Event<'_>) -> (Option>, Option>) { - match event { - Event::Start(Tag::Table(_)) => ( - Some(Event::Html(r#"
"#.into())), - Some(event), - ), - Event::End(TagEnd::Table) => (Some(event), Some(Event::Html(r#"
"#.into()))), - _ => (Some(event), None), - } -} - -fn clean_codeblock_headers(event: Event<'_>) -> Event<'_> { - match event { - Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(ref info))) => { - let info: String = info - .chars() - .map(|x| match x { - ' ' | '\t' => ',', - _ => x, - }) - .filter(|ch| !ch.is_whitespace()) - .collect(); - - Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(CowStr::from(info)))) - } - _ => event, - } -} - -/// Prints a "backtrace" of some `Error`. -pub fn log_backtrace(e: &Error) { - error!("Error: {}", e); - - for cause in e.chain().skip(1) { - error!("\tCaused By: {}", cause); - } -} - -pub(crate) fn special_escape(mut s: &str) -> String { - let mut escaped = String::with_capacity(s.len()); - let needs_escape: &[char] = &['<', '>', '\'', '\\', '&']; - while let Some(next) = s.find(needs_escape) { - escaped.push_str(&s[..next]); - match s.as_bytes()[next] { - b'<' => escaped.push_str("<"), - b'>' => escaped.push_str(">"), - b'\'' => escaped.push_str("'"), - b'\\' => escaped.push_str("\"), - b'&' => escaped.push_str("&"), - _ => unreachable!(), - } - s = &s[next + 1..]; - } - escaped.push_str(s); - escaped -} - -pub(crate) fn bracket_escape(mut s: &str) -> String { - let mut escaped = String::with_capacity(s.len()); - let needs_escape: &[char] = &['<', '>']; - while let Some(next) = s.find(needs_escape) { - escaped.push_str(&s[..next]); - match s.as_bytes()[next] { - b'<' => escaped.push_str("<"), - b'>' => escaped.push_str(">"), - _ => unreachable!(), - } - s = &s[next + 1..]; - } - escaped.push_str(s); - escaped -} - -#[cfg(test)] -mod tests { - use super::{bracket_escape, special_escape}; - - mod render_markdown { - use super::super::render_markdown; - - #[test] - fn preserves_external_links() { - assert_eq!( - render_markdown("[example](https://www.rust-lang.org/)", false), - "

example

\n" - ); - } - - #[test] - fn it_can_adjust_markdown_links() { - assert_eq!( - render_markdown("[example](example.md)", false), - "

example

\n" - ); - assert_eq!( - render_markdown("[example_anchor](example.md#anchor)", false), - "

example_anchor

\n" - ); - - // this anchor contains 'md' inside of it - assert_eq!( - render_markdown("[phantom data](foo.html#phantomdata)", false), - "

phantom data

\n" - ); - } - - #[test] - fn it_can_wrap_tables() { - let src = r#" -| Original | Punycode | Punycode + Encoding | -|-----------------|-----------------|---------------------| -| føø | f-5gaa | f_5gaa | -"#; - let out = r#" -
- -
OriginalPunycodePunycode + Encoding
føøf-5gaaf_5gaa
-
-"#.trim(); - assert_eq!(render_markdown(src, false), out); - } - - #[test] - fn it_can_keep_quotes_straight() { - assert_eq!(render_markdown("'one'", false), "

'one'

\n"); - } - - #[test] - fn it_can_make_quotes_curly_except_when_they_are_in_code() { - let input = r#" -'one' -``` -'two' -``` -`'three'` 'four'"#; - let expected = r#"

‘one’

-
'two'
-
-

'three' ‘four’

-"#; - assert_eq!(render_markdown(input, true), expected); - } - - #[test] - fn whitespace_outside_of_codeblock_header_is_preserved() { - let input = r#" -some text with spaces -```rust -fn main() { -// code inside is unchanged -} -``` -more text with spaces -"#; - - let expected = r#"

some text with spaces

-
fn main() {
-// code inside is unchanged
-}
-
-

more text with spaces

-"#; - assert_eq!(render_markdown(input, false), expected); - assert_eq!(render_markdown(input, true), expected); - } - - #[test] - fn rust_code_block_properties_are_passed_as_space_delimited_class() { - let input = r#" -```rust,no_run,should_panic,property_3 -``` -"#; - - let expected = r#"
-"#; - assert_eq!(render_markdown(input, false), expected); - assert_eq!(render_markdown(input, true), expected); - } - - #[test] - fn rust_code_block_properties_with_whitespace_are_passed_as_space_delimited_class() { - let input = r#" -```rust, no_run,,,should_panic , ,property_3 -``` -"#; - - let expected = r#"
-"#; - assert_eq!(render_markdown(input, false), expected); - assert_eq!(render_markdown(input, true), expected); - } - - #[test] - fn rust_code_block_without_properties_has_proper_html_class() { - let input = r#" -```rust -``` -"#; - - let expected = r#"
-"#; - assert_eq!(render_markdown(input, false), expected); - assert_eq!(render_markdown(input, true), expected); - - let input = r#" -```rust -``` -"#; - assert_eq!(render_markdown(input, false), expected); - assert_eq!(render_markdown(input, true), expected); - } - } - - #[allow(deprecated)] - mod id_from_content { - use super::super::id_from_content; - - #[test] - fn it_generates_anchors() { - assert_eq!( - id_from_content("## Method-call expressions"), - "method-call-expressions" - ); - assert_eq!(id_from_content("## **Bold** title"), "bold-title"); - assert_eq!(id_from_content("## `Code` title"), "code-title"); - assert_eq!( - id_from_content("## title foo"), - "title-foo" - ); - } - - #[test] - fn it_generates_anchors_from_non_ascii_initial() { - assert_eq!( - id_from_content("## `--passes`: add more rustdoc passes"), - "--passes-add-more-rustdoc-passes" - ); - assert_eq!( - id_from_content("## 中文標題 CJK title"), - "中文標題-cjk-title" - ); - assert_eq!(id_from_content("## Über"), "Über"); - } - } - - mod html_munging { - use super::super::{normalize_id, unique_id_from_content}; - - #[test] - fn it_normalizes_ids() { - assert_eq!( - normalize_id("`--passes`: add more rustdoc passes"), - "--passes-add-more-rustdoc-passes" - ); - assert_eq!( - normalize_id("Method-call 🐙 expressions \u{1f47c}"), - "method-call--expressions-" - ); - assert_eq!(normalize_id("_-_12345"), "_-_12345"); - assert_eq!(normalize_id("12345"), "12345"); - assert_eq!(normalize_id("中文"), "中文"); - assert_eq!(normalize_id("にほんご"), "にほんご"); - assert_eq!(normalize_id("한국어"), "한국어"); - assert_eq!(normalize_id(""), ""); - } - - #[test] - fn it_generates_unique_ids_from_content() { - // Same id if not given shared state - assert_eq!( - unique_id_from_content("## 中文標題 CJK title", &mut Default::default()), - "中文標題-cjk-title" - ); - assert_eq!( - unique_id_from_content("## 中文標題 CJK title", &mut Default::default()), - "中文標題-cjk-title" - ); - - // Different id if given shared state - let mut id_counter = Default::default(); - assert_eq!(unique_id_from_content("## Über", &mut id_counter), "Über"); - assert_eq!( - unique_id_from_content("## 中文標題 CJK title", &mut id_counter), - "中文標題-cjk-title" - ); - assert_eq!(unique_id_from_content("## Über", &mut id_counter), "Über-1"); - assert_eq!(unique_id_from_content("## Über", &mut id_counter), "Über-2"); - } - } - - #[test] - fn escaped_brackets() { - assert_eq!(bracket_escape(""), ""); - assert_eq!(bracket_escape("<"), "<"); - assert_eq!(bracket_escape(">"), ">"); - assert_eq!(bracket_escape("<>"), "<>"); - assert_eq!(bracket_escape(""), "<test>"); - assert_eq!(bracket_escape("ab"), "a<test>b"); - assert_eq!(bracket_escape("'"), "'"); - assert_eq!(bracket_escape("\\"), "\\"); - } - - #[test] - fn escaped_special() { - assert_eq!(special_escape(""), ""); - assert_eq!(special_escape("<"), "<"); - assert_eq!(special_escape(">"), ">"); - assert_eq!(special_escape("<>"), "<>"); - assert_eq!(special_escape(""), "<test>"); - assert_eq!(special_escape("ab"), "a<test>b"); - assert_eq!(special_escape("'"), "'"); - assert_eq!(special_escape("\\"), "\"); - assert_eq!(special_escape("&"), "&"); - } -} diff --git a/tests/testsuite/config.rs b/tests/testsuite/config.rs index f5b14875a2..2170b2ee46 100644 --- a/tests/testsuite/config.rs +++ b/tests/testsuite/config.rs @@ -201,7 +201,7 @@ ERROR Invalid configuration file | 2 | title = "bad-config" | ^^^^^ -unknown field `title`, expected `edition` +unknown field `title`, expected `manifest` or `edition` "#]]);