Move highlighting logic from JS to Rust

Continue migrating JS functionality

Cleanup

Fix compile error

Clean up the diff

Set toggle font to sans-serif

Author: willcrichton

src/librustdoc/doctest.rs

@@ -45,7 +45,7 @@ crate struct TestOptions {
     crate attrs: Vec<String>,
 }
 
-crate fn make_rustc_config(options: &Options) -> interface::Config {
+crate fn run(options: Options) -> Result<(), ErrorReported> {
     let input = config::Input::File(options.input.clone());
 
     let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
@@ -87,7 +87,7 @@ crate fn make_rustc_config(options: &Options) -> interface::Config {
     let mut cfgs = options.cfgs.clone();
     cfgs.push("doc".to_owned());
     cfgs.push("doctest".to_owned());
-    interface::Config {
+    let config = interface::Config {
         opts: sessopts,
         crate_cfg: interface::parse_cfgspecs(cfgs),
         input,
@@ -103,11 +103,7 @@ crate fn make_rustc_config(options: &Options) -> interface::Config {
         override_queries: None,
         make_codegen_backend: None,
         registry: rustc_driver::diagnostics_registry(),
-    }
-}
-
-crate fn run(options: Options) -> Result<(), ErrorReported> {
-    let config = make_rustc_config(&options);
+    };
 
     let test_args = options.test_args.clone();
     let display_doctest_warnings = options.display_doctest_warnings;

src/librustdoc/html/highlight.rs

@@ -12,6 +12,7 @@ use crate::html::render::Context;
 use std::collections::VecDeque;
 use std::fmt::{Display, Write};
 
+use rustc_data_structures::fx::FxHashMap;
 use rustc_lexer::{LiteralKind, TokenKind};
 use rustc_span::edition::Edition;
 use rustc_span::symbol::Symbol;
@@ -30,6 +31,8 @@ crate struct ContextInfo<'a, 'b, 'c> {
     crate root_path: &'c str,
 }
 
+crate type DecorationInfo = FxHashMap<&'static str, Vec<(u32, u32)>>;
+
 /// Highlights `src`, returning the HTML output.
 crate fn render_with_highlighting(
     src: &str,
@@ -40,6 +43,7 @@ crate fn render_with_highlighting(
     edition: Edition,
     extra_content: Option<Buffer>,
     context_info: Option<ContextInfo<'_, '_, '_>>,
+    decoration_info: Option<DecorationInfo>,
 ) {
     debug!("highlighting: ================\n{}\n==============", src);
     if let Some((edition_info, class)) = tooltip {
@@ -56,7 +60,7 @@ crate fn render_with_highlighting(
     }
 
     write_header(out, class, extra_content);
-    write_code(out, &src, edition, context_info);
+    write_code(out, &src, edition, context_info, decoration_info);
     write_footer(out, playground_button);
 }
 
@@ -89,17 +93,23 @@ fn write_code(
     src: &str,
     edition: Edition,
     context_info: Option<ContextInfo<'_, '_, '_>>,
+    decoration_info: Option<DecorationInfo>,
 ) {
     // This replace allows to fix how the code source with DOS backline characters is displayed.
     let src = src.replace("\r\n", "\n");
-    Classifier::new(&src, edition, context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP))
-        .highlight(&mut |highlight| {
-            match highlight {
-                Highlight::Token { text, class } => string(out, Escape(text), class, &context_info),
-                Highlight::EnterSpan { class } => enter_span(out, class),
-                Highlight::ExitSpan => exit_span(out),
-            };
-        });
+    Classifier::new(
+        &src,
+        edition,
+        context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP),
+        decoration_info,
+    )
+    .highlight(&mut |highlight| {
+        match highlight {
+            Highlight::Token { text, class } => string(out, Escape(text), class, &context_info),
+            Highlight::EnterSpan { class } => enter_span(out, class),
+            Highlight::ExitSpan => exit_span(out),
+        };
+    });
 }
 
 fn write_footer(out: &mut Buffer, playground_button: Option<&str>) {
@@ -127,6 +137,7 @@ enum Class {
     PreludeTy,
     PreludeVal,
     QuestionMark,
+    Decoration(&'static str),
 }
 
 impl Class {
@@ -150,6 +161,7 @@ impl Class {
             Class::PreludeTy => "prelude-ty",
             Class::PreludeVal => "prelude-val",
             Class::QuestionMark => "question-mark",
+            Class::Decoration(kind) => kind,
         }
     }
 
@@ -244,7 +256,28 @@ impl Iterator for PeekIter<'a> {
     type Item = (TokenKind, &'a str);
     fn next(&mut self) -> Option<Self::Item> {
         self.peek_pos = 0;
-        if let Some(first) = self.stored.pop_front() { Some(first) } else { self.iter.next() }
+        if let Some(first) = self.stored.pop_front() {
+            Some(first)
+        } else {
+            self.iter.next()
+        }
+    }
+}
+
+/// Custom spans inserted into the source. Eg --scrape-examples uses this to highlight function calls
+struct Decorations {
+    starts: Vec<(u32, &'static str)>,
+    ends: Vec<u32>,
+}
+
+impl Decorations {
+    fn new(info: DecorationInfo) -> Self {
+        let (starts, ends) = info
+            .into_iter()
+            .map(|(kind, ranges)| ranges.into_iter().map(move |(lo, hi)| ((lo, kind), hi)))
+            .flatten()
+            .unzip();
+        Decorations { starts, ends }
     }
 }
 
@@ -259,12 +292,18 @@ struct Classifier<'a> {
     byte_pos: u32,
     file_span: Span,
     src: &'a str,
+    decorations: Option<Decorations>,
 }
 
 impl<'a> Classifier<'a> {
     /// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
     /// file span which will be used later on by the `span_correspondance_map`.
-    fn new(src: &str, edition: Edition, file_span: Span) -> Classifier<'_> {
+    fn new(
+        src: &str,
+        edition: Edition,
+        file_span: Span,
+        decoration_info: Option<DecorationInfo>,
+    ) -> Classifier<'_> {
         let tokens = PeekIter::new(TokenIter { src });
         Classifier {
             tokens,
@@ -275,6 +314,7 @@ impl<'a> Classifier<'a> {
             byte_pos: 0,
             file_span,
             src,
+            decorations,
         }
     }
 
@@ -356,6 +396,19 @@ impl<'a> Classifier<'a> {
     /// token is used.
     fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'a>)) {
         loop {
+            if let Some(decs) = self.decorations.as_mut() {
+                let byte_pos = self.byte_pos;
+                let n_starts = decs.starts.iter().filter(|(i, _)| byte_pos >= *i).count();
+                for (_, kind) in decs.starts.drain(0..n_starts) {
+                    sink(Highlight::EnterSpan { class: Class::Decoration(kind) });
+                }
+
+                let n_ends = decs.ends.iter().filter(|i| byte_pos >= **i).count();
+                for _ in decs.ends.drain(0..n_ends) {
+                    sink(Highlight::ExitSpan);
+                }
+            }
+
             if self
                 .tokens
                 .peek()

src/librustdoc/html/highlight/tests.rs

@@ -22,7 +22,7 @@ fn test_html_highlighting() {
         let src = include_str!("fixtures/sample.rs");
         let html = {
             let mut out = Buffer::new();
-            write_code(&mut out, src, Edition::Edition2018, None);
+            write_code(&mut out, src, Edition::Edition2018, None, None);
             format!("{}<pre><code>{}</code></pre>\n", STYLE, out.into_inner())
         };
         expect_file!["fixtures/sample.html"].assert_eq(&html);
@@ -36,7 +36,7 @@ fn test_dos_backline() {
     println!(\"foo\");\r\n\
 }\r\n";
         let mut html = Buffer::new();
-        write_code(&mut html, src, Edition::Edition2018, None);
+        write_code(&mut html, src, Edition::Edition2018, None, None);
         expect_file!["fixtures/dos_line.html"].assert_eq(&html.into_inner());
     });
 }
@@ -50,7 +50,7 @@ let x = super::b::foo;
 let y = Self::whatever;";
 
         let mut html = Buffer::new();
-        write_code(&mut html, src, Edition::Edition2018, None);
+        write_code(&mut html, src, Edition::Edition2018, None, None);
         expect_file!["fixtures/highlight.html"].assert_eq(&html.into_inner());
     });
 }

src/librustdoc/html/markdown.rs

@@ -360,6 +360,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
             edition,
             None,
             None,
+            None,
         );
         Some(Event::Html(s.into_inner().into()))
     }

src/librustdoc/html/render/mod.rs

@@ -46,7 +46,7 @@ use std::string::ToString;
 
 use rustc_ast_pretty::pprust;
 use rustc_attr::{ConstStability, Deprecation, StabilityLevel};
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir as hir;
 use rustc_hir::def::CtorKind;
 use rustc_hir::def_id::DefId;
@@ -2496,23 +2496,28 @@ fn render_call_locations(
 
         // To reduce file sizes, we only want to embed the source code needed to understand the example, not
         // the entire file. So we find the smallest byte range that covers all items enclosing examples.
+        assert!(call_data.locations.len() > 0);
         let min_loc =
-            call_data.locations.iter().min_by_key(|loc| loc.enclosing_item_span.0).unwrap();
-        let min_byte = min_loc.enclosing_item_span.0;
-        let min_line = min_loc.enclosing_item_lines.0;
+            call_data.locations.iter().min_by_key(|loc| loc.enclosing_item.byte_span.0).unwrap();
+        let min_byte = min_loc.enclosing_item.byte_span.0;
+        let min_line = min_loc.enclosing_item.line_span.0;
         let max_byte =
-            call_data.locations.iter().map(|loc| loc.enclosing_item_span.1).max().unwrap();
+            call_data.locations.iter().map(|loc| loc.enclosing_item.byte_span.1).max().unwrap();
 
         // The output code is limited to that byte range.
-        let contents_subset = &contents[min_byte..max_byte];
+        let contents_subset = &contents[(min_byte as usize)..(max_byte as usize)];
 
         // The call locations need to be updated to reflect that the size of the program has changed.
         // Specifically, the ranges are all subtracted by `min_byte` since that's the new zero point.
-        let locations = call_data
+        let (byte_ranges, line_ranges): (Vec<_>, Vec<_>) = call_data
             .locations
             .iter()
-            .map(|loc| (loc.call_span.0 - min_byte, loc.call_span.1 - min_byte))
-            .collect::<Vec<_>>();
+            .map(|loc| {
+                let (byte_lo, byte_hi) = loc.call_expr.byte_span;
+                let (line_lo, line_hi) = loc.call_expr.line_span;
+                ((byte_lo - min_byte, byte_hi - min_byte), (line_lo - min_line, line_hi - min_line))
+            })
+            .unzip();
 
         let edition = cx.shared.edition();
         write!(
@@ -2524,15 +2529,26 @@ fn render_call_locations(
             // The code and locations are encoded as data attributes, so they can be read
             // later by the JS for interactions.
             code = contents_subset.replace("\"", "&quot;"),
-            locations = serde_json::to_string(&locations).unwrap(),
+            locations = serde_json::to_string(&line_ranges).unwrap(),
         );
         write!(w, r#"<span class="prev">&pr;</span> <span class="next">&sc;</span>"#);
         write!(w, r#"<span class="expand">&varr;</span>"#);
 
         // FIXME(wcrichto): where should file_span and root_path come from?
         let file_span = rustc_span::DUMMY_SP;
         let root_path = "".to_string();
-        sources::print_src(w, contents_subset, edition, file_span, cx, &root_path, Some(min_line));
+        let mut decoration_info = FxHashMap::default();
+        decoration_info.insert("highlight", byte_ranges);
+        sources::print_src(
+            w,
+            contents_subset,
+            edition,
+            file_span,
+            cx,
+            &root_path,
+            Some(min_line),
+            Some(decoration_info),
+        );
         write!(w, "</div></div>");
     };
 
@@ -2542,7 +2558,8 @@ fn render_call_locations(
     // understand at a glance.
     let ordered_locations = {
         let sort_criterion = |(_, call_data): &(_, &CallData)| {
-            let (lo, hi) = call_data.locations[0].enclosing_item_span;
+            // Use the first location because that's what the user will see initially
+            let (lo, hi) = call_data.locations[0].enclosing_item.byte_span;
             hi - lo
         };
 

src/librustdoc/html/render/print_item.rs

@@ -1117,6 +1117,7 @@ fn item_macro(w: &mut Buffer, cx: &Context<'_>, it: &clean::Item, t: &clean::Mac
             it.span(cx.tcx()).inner().edition(),
             None,
             None,
+            None,
         );
     });
     document(w, cx, it, None, HeadingOffset::H2)

src/librustdoc/html/sources.rs

@@ -212,6 +212,7 @@ impl SourceCollector<'_, 'tcx> {
                     &self.cx,
                     &root_path,
                     None,
+                    None,
                 )
             },
             &self.cx.shared.style_files,
@@ -259,6 +260,7 @@ crate fn print_src(
     context: &Context<'_>,
     root_path: &str,
     offset: Option<usize>,
+    decoration_info: Option<highlight::DecorationInfo>,
 ) {
     let lines = s.lines().count();
     let mut line_numbers = Buffer::empty_from(buf);
@@ -283,5 +285,6 @@ crate fn print_src(
         edition,
         Some(line_numbers),
         Some(highlight::ContextInfo { context, file_span, root_path }),
+        decoration_info,
     );
 }