Auto merge of rust-lang#60065 - QuietMisdreavus:async-move-doctests, r=ollie27

rustdoc: set the default edition when pre-parsing a doctest

Fixes rust-lang#59313 (possibly more? i think we've had issues with parsing edition-specific syntax in doctests at some point)

When handling a doctest, rustdoc needs to parse it beforehand, so that it can see whether it declares a `fn main` or `extern crate my_crate` explicitly. However, while doing this, rustdoc doesn't set the "default edition" used by the parser like the regular compilation runs do. This caused a problem when parsing a doctest with an `async move` block in it, since it was expecting the `move` keyword to start a closure, not a block.

This PR changes the `rustdoc::test::make_test` function to set the parser's default edition while looking for a main function and `extern crate` statement. However, to do this, `make_test` needs to know what edition to set. Since this is also used during the HTML rendering process (to make playground URLs), now the HTML renderer needs to know about the default edition. Upshot: rendering standalone markdown files can now accept a "default edition" for their doctests with the `--edition` flag! (I'm pretty sure i waffled around how to set that a long time ago when we first added the `--edition` flag... `>_>`)

I'm posting this before i stop for the night so that i can write this description while it's still in my head, but before this merges i want to make sure that (1) the `rustdoc-ui/failed-doctest-output` test still works (i expect it doesn't), and (2) i add a test with the sample from the linked issue.

Author: bors

src/librustdoc/config.rs

@@ -13,7 +13,7 @@ use rustc::session::config::{nightly_options, build_codegen_options, build_debug
 use rustc::session::search_paths::SearchPath;
 use rustc_driver;
 use rustc_target::spec::TargetTriple;
-use syntax::edition::Edition;
+use syntax::edition::{Edition, DEFAULT_EDITION};
 
 use crate::core::new_handler;
 use crate::externalfiles::ExternalHtml;
@@ -386,27 +386,31 @@ impl Options {
             }
         }
 
+        let edition = if let Some(e) = matches.opt_str("edition") {
+            match e.parse() {
+                Ok(e) => e,
+                Err(_) => {
+                    diag.struct_err("could not parse edition").emit();
+                    return Err(1);
+                }
+            }
+        } else {
+            DEFAULT_EDITION
+        };
+
         let mut id_map = html::markdown::IdMap::new();
         id_map.populate(html::render::initial_ids());
         let external_html = match ExternalHtml::load(
                 &matches.opt_strs("html-in-header"),
                 &matches.opt_strs("html-before-content"),
                 &matches.opt_strs("html-after-content"),
                 &matches.opt_strs("markdown-before-content"),
-                &matches.opt_strs("markdown-after-content"), &diag, &mut id_map) {
+                &matches.opt_strs("markdown-after-content"),
+                &diag, &mut id_map, edition) {
             Some(eh) => eh,
             None => return Err(3),
         };
 
-        let edition = matches.opt_str("edition").unwrap_or("2015".to_string());
-        let edition = match edition.parse() {
-            Ok(e) => e,
-            Err(_) => {
-                diag.struct_err("could not parse edition").emit();
-                return Err(1);
-            }
-        };
-
         match matches.opt_str("r").as_ref().map(|s| &**s) {
             Some("rust") | None => {}
             Some(s) => {

src/librustdoc/externalfiles.rs

@@ -3,6 +3,7 @@ use std::path::Path;
 use std::str;
 use errors;
 use crate::syntax::feature_gate::UnstableFeatures;
+use crate::syntax::edition::Edition;
 use crate::html::markdown::{IdMap, ErrorCodes, Markdown};
 
 use std::cell::RefCell;
@@ -23,7 +24,7 @@ pub struct ExternalHtml {
 impl ExternalHtml {
     pub fn load(in_header: &[String], before_content: &[String], after_content: &[String],
                 md_before_content: &[String], md_after_content: &[String], diag: &errors::Handler,
-                id_map: &mut IdMap)
+                id_map: &mut IdMap, edition: Edition)
             -> Option<ExternalHtml> {
         let codes = ErrorCodes::from(UnstableFeatures::from_environment().is_nightly_build());
         load_external_files(in_header, diag)
@@ -34,7 +35,8 @@ impl ExternalHtml {
             .and_then(|(ih, bc)|
                 load_external_files(md_before_content, diag)
                     .map(|m_bc| (ih,
-                            format!("{}{}", bc, Markdown(&m_bc, &[], RefCell::new(id_map), codes))))
+                            format!("{}{}", bc, Markdown(&m_bc, &[], RefCell::new(id_map),
+                                    codes, edition))))
             )
             .and_then(|(ih, bc)|
                 load_external_files(after_content, diag)
@@ -43,7 +45,8 @@ impl ExternalHtml {
             .and_then(|(ih, bc, ac)|
                 load_external_files(md_after_content, diag)
                     .map(|m_ac| (ih, bc,
-                            format!("{}{}", ac, Markdown(&m_ac, &[], RefCell::new(id_map), codes))))
+                            format!("{}{}", ac, Markdown(&m_ac, &[], RefCell::new(id_map),
+                                    codes, edition))))
             )
             .map(|(ih, bc, ac)|
                 ExternalHtml {

src/librustdoc/html/markdown.rs

@@ -8,12 +8,16 @@
 //! ```
 //! #![feature(rustc_private)]
 //!
+//! extern crate syntax;
+//!
+//! use syntax::edition::Edition;
 //! use rustdoc::html::markdown::{IdMap, Markdown, ErrorCodes};
 //! use std::cell::RefCell;
 //!
 //! let s = "My *markdown* _text_";
 //! let mut id_map = IdMap::new();
-//! let html = format!("{}", Markdown(s, &[], RefCell::new(&mut id_map), ErrorCodes::Yes));
+//! let html = format!("{}", Markdown(s, &[], RefCell::new(&mut id_map),
+//!                                   ErrorCodes::Yes, Edition::Edition2015));
 //! // ... something using html
 //! ```
 
@@ -42,14 +46,21 @@ fn opts() -> Options {
 /// A unit struct which has the `fmt::Display` trait implemented. When
 /// formatted, this struct will emit the HTML corresponding to the rendered
 /// version of the contained markdown string.
-/// The second parameter is a list of link replacements
+///
+/// The second parameter is a list of link replacements.
+///
+/// The third is the current list of used header IDs.
+///
+/// The fourth is whether to allow the use of explicit error codes in doctest lang strings.
+///
+/// The fifth is what default edition to use when parsing doctests (to add a `fn main`).
 pub struct Markdown<'a>(
-    pub &'a str, pub &'a [(String, String)], pub RefCell<&'a mut IdMap>, pub ErrorCodes);
+    pub &'a str, pub &'a [(String, String)], pub RefCell<&'a mut IdMap>, pub ErrorCodes, pub Edition);
 /// A unit struct like `Markdown`, that renders the markdown with a
 /// table of contents.
-pub struct MarkdownWithToc<'a>(pub &'a str, pub RefCell<&'a mut IdMap>, pub ErrorCodes);
+pub struct MarkdownWithToc<'a>(pub &'a str, pub RefCell<&'a mut IdMap>, pub ErrorCodes, pub Edition);
 /// A unit struct like `Markdown`, that renders the markdown escaping HTML tags.
-pub struct MarkdownHtml<'a>(pub &'a str, pub RefCell<&'a mut IdMap>, pub ErrorCodes);
+pub struct MarkdownHtml<'a>(pub &'a str, pub RefCell<&'a mut IdMap>, pub ErrorCodes, pub Edition);
 /// A unit struct like `Markdown`, that renders only the first paragraph.
 pub struct MarkdownSummaryLine<'a>(pub &'a str, pub &'a [(String, String)]);
 
@@ -146,13 +157,15 @@ thread_local!(pub static PLAYGROUND: RefCell<Option<(Option<String>, String)>> =
 struct CodeBlocks<'a, I: Iterator<Item = Event<'a>>> {
     inner: I,
     check_error_codes: ErrorCodes,
+    edition: Edition,
 }
 
 impl<'a, I: Iterator<Item = Event<'a>>> CodeBlocks<'a, I> {
-    fn new(iter: I, error_codes: ErrorCodes) -> Self {
+    fn new(iter: I, error_codes: ErrorCodes, edition: Edition) -> Self {
         CodeBlocks {
             inner: iter,
             check_error_codes: error_codes,
+            edition,
         }
     }
 }
@@ -177,6 +190,9 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'a, I> {
             return event;
         }
 
+        let explicit_edition = edition.is_some();
+        let edition = edition.unwrap_or(self.edition);
+
         let mut origtext = String::new();
         for event in &mut self.inner {
             match event {
@@ -202,22 +218,14 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'a, I> {
                     .collect::<Vec<Cow<'_, str>>>().join("\n");
                 let krate = krate.as_ref().map(|s| &**s);
                 let (test, _) = test::make_test(&test, krate, false,
-                                           &Default::default());
+                                           &Default::default(), edition);
                 let channel = if test.contains("#![feature(") {
                     "&amp;version=nightly"
                 } else {
                     ""
                 };
 
-                let edition_string = if let Some(e @ Edition::Edition2018) = edition {
-                    format!("&amp;edition={}{}", e,
-                            if channel == "&amp;version=nightly" { "" }
-                            else { "&amp;version=nightly" })
-                } else if let Some(e) = edition {
-                    format!("&amp;edition={}", e)
-                } else {
-                    "".to_owned()
-                };
+                let edition_string = format!("&amp;edition={}", edition);
 
                 // These characters don't need to be escaped in a URI.
                 // FIXME: use a library function for percent encoding.
@@ -247,8 +255,8 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'a, I> {
                 Some(("This example is not tested".to_owned(), "ignore"))
             } else if compile_fail {
                 Some(("This example deliberately fails to compile".to_owned(), "compile_fail"))
-            } else if let Some(e) = edition {
-                Some((format!("This code runs with edition {}", e), "edition"))
+            } else if explicit_edition {
+                Some((format!("This code runs with edition {}", edition), "edition"))
             } else {
                 None
             };
@@ -259,7 +267,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'a, I> {
                     Some(&format!("rust-example-rendered{}",
                                   if ignore { " ignore" }
                                   else if compile_fail { " compile_fail" }
-                                  else if edition.is_some() { " edition " }
+                                  else if explicit_edition { " edition " }
                                   else { "" })),
                     playground_button.as_ref().map(String::as_str),
                     Some((s1.as_str(), s2))));
@@ -270,7 +278,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'a, I> {
                     Some(&format!("rust-example-rendered{}",
                                   if ignore { " ignore" }
                                   else if compile_fail { " compile_fail" }
-                                  else if edition.is_some() { " edition " }
+                                  else if explicit_edition { " edition " }
                                   else { "" })),
                     playground_button.as_ref().map(String::as_str),
                     None));
@@ -659,7 +667,7 @@ impl LangString {
 
 impl<'a> fmt::Display for Markdown<'a> {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let Markdown(md, links, ref ids, codes) = *self;
+        let Markdown(md, links, ref ids, codes, edition) = *self;
         let mut ids = ids.borrow_mut();
 
         // This is actually common enough to special-case
@@ -678,7 +686,7 @@ impl<'a> fmt::Display for Markdown<'a> {
 
         let p = HeadingLinks::new(p, None, &mut ids);
         let p = LinkReplacer::new(p, links);
-        let p = CodeBlocks::new(p, codes);
+        let p = CodeBlocks::new(p, codes, edition);
         let p = Footnotes::new(p);
         html::push_html(&mut s, p);
 
@@ -688,7 +696,7 @@ impl<'a> fmt::Display for Markdown<'a> {
 
 impl<'a> fmt::Display for MarkdownWithToc<'a> {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let MarkdownWithToc(md, ref ids, codes) = *self;
+        let MarkdownWithToc(md, ref ids, codes, edition) = *self;
         let mut ids = ids.borrow_mut();
 
         let p = Parser::new_ext(md, opts());
@@ -699,7 +707,7 @@ impl<'a> fmt::Display for MarkdownWithToc<'a> {
 
         {
             let p = HeadingLinks::new(p, Some(&mut toc), &mut ids);
-            let p = CodeBlocks::new(p, codes);
+            let p = CodeBlocks::new(p, codes, edition);
             let p = Footnotes::new(p);
             html::push_html(&mut s, p);
         }
@@ -712,7 +720,7 @@ impl<'a> fmt::Display for MarkdownWithToc<'a> {
 
 impl<'a> fmt::Display for MarkdownHtml<'a> {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let MarkdownHtml(md, ref ids, codes) = *self;
+        let MarkdownHtml(md, ref ids, codes, edition) = *self;
         let mut ids = ids.borrow_mut();
 
         // This is actually common enough to special-case
@@ -728,7 +736,7 @@ impl<'a> fmt::Display for MarkdownHtml<'a> {
         let mut s = String::with_capacity(md.len() * 3 / 2);
 
         let p = HeadingLinks::new(p, None, &mut ids);
-        let p = CodeBlocks::new(p, codes);
+        let p = CodeBlocks::new(p, codes, edition);
         let p = Footnotes::new(p);
         html::push_html(&mut s, p);
 
@@ -1046,7 +1054,7 @@ mod tests {
     use super::{ErrorCodes, LangString, Markdown, MarkdownHtml, IdMap};
     use super::plain_summary_line;
     use std::cell::RefCell;
-    use syntax::edition::Edition;
+    use syntax::edition::{Edition, DEFAULT_EDITION};
 
     #[test]
     fn test_lang_string_parse() {
@@ -1098,7 +1106,8 @@ mod tests {
     fn test_header() {
         fn t(input: &str, expect: &str) {
             let mut map = IdMap::new();
-            let output = Markdown(input, &[], RefCell::new(&mut map), ErrorCodes::Yes).to_string();
+            let output = Markdown(input, &[], RefCell::new(&mut map),
+                                  ErrorCodes::Yes, DEFAULT_EDITION).to_string();
             assert_eq!(output, expect, "original: {}", input);
         }
 
@@ -1120,7 +1129,8 @@ mod tests {
     fn test_header_ids_multiple_blocks() {
         let mut map = IdMap::new();
         fn t(map: &mut IdMap, input: &str, expect: &str) {
-            let output = Markdown(input, &[], RefCell::new(map), ErrorCodes::Yes).to_string();
+            let output = Markdown(input, &[], RefCell::new(map),
+                                  ErrorCodes::Yes, DEFAULT_EDITION).to_string();
             assert_eq!(output, expect, "original: {}", input);
         }
 
@@ -1157,7 +1167,8 @@ mod tests {
     fn test_markdown_html_escape() {
         fn t(input: &str, expect: &str) {
             let mut idmap = IdMap::new();
-            let output = MarkdownHtml(input, RefCell::new(&mut idmap), ErrorCodes::Yes).to_string();
+            let output = MarkdownHtml(input, RefCell::new(&mut idmap),
+                                      ErrorCodes::Yes, DEFAULT_EDITION).to_string();
             assert_eq!(output, expect, "original: {}", input);
         }
 

src/librustdoc/html/render.rs

@@ -47,6 +47,7 @@ use std::rc::Rc;
 use errors;
 use serialize::json::{ToJson, Json, as_json};
 use syntax::ast;
+use syntax::edition::Edition;
 use syntax::ext::base::MacroKind;
 use syntax::source_map::FileName;
 use syntax::feature_gate::UnstableFeatures;
@@ -108,6 +109,8 @@ struct Context {
     /// publicly reused items to redirect to the right location.
     pub render_redirect_pages: bool,
     pub codes: ErrorCodes,
+    /// The default edition used to parse doctests.
+    pub edition: Edition,
     /// The map used to ensure all generated 'id=' attributes are unique.
     id_map: Rc<RefCell<IdMap>>,
     pub shared: Arc<SharedContext>,
@@ -514,7 +517,8 @@ pub fn run(mut krate: clean::Crate,
            options: RenderOptions,
            passes: FxHashSet<String>,
            renderinfo: RenderInfo,
-           diag: &errors::Handler) -> Result<(), Error> {
+           diag: &errors::Handler,
+           edition: Edition) -> Result<(), Error> {
     // need to save a copy of the options for rendering the index page
     let md_opts = options.clone();
     let RenderOptions {
@@ -604,6 +608,7 @@ pub fn run(mut krate: clean::Crate,
         dst,
         render_redirect_pages: false,
         codes: ErrorCodes::from(UnstableFeatures::from_environment().is_nightly_build()),
+        edition,
         id_map: Rc::new(RefCell::new(id_map)),
         shared: Arc::new(scx),
     };
@@ -1128,7 +1133,7 @@ themePicker.onblur = handleThemeButtonsBlur;
             md_opts.output = cx.dst.clone();
             md_opts.external_html = (*cx.shared).layout.external_html.clone();
 
-            crate::markdown::render(index_page, md_opts, diag);
+            crate::markdown::render(index_page, md_opts, diag, cx.edition);
         } else {
             let dst = cx.dst.join("index.html");
             let mut w = BufWriter::new(try_err!(File::create(&dst), &dst));
@@ -2553,7 +2558,7 @@ fn render_markdown(w: &mut fmt::Formatter<'_>,
            if is_hidden { " hidden" } else { "" },
            prefix,
            Markdown(md_text, &links, RefCell::new(&mut ids),
-           cx.codes))
+           cx.codes, cx.edition))
 }
 
 fn document_short(
@@ -2918,7 +2923,7 @@ fn short_stability(item: &clean::Item, cx: &Context) -> Vec<String> {
 
         if let Some(note) = note {
             let mut ids = cx.id_map.borrow_mut();
-            let html = MarkdownHtml(&note, RefCell::new(&mut ids), error_codes);
+            let html = MarkdownHtml(&note, RefCell::new(&mut ids), error_codes, cx.edition);
             message.push_str(&format!(": {}", html));
         }
         stability.push(format!("<div class='stab deprecated'>{}</div>", message));
@@ -2967,7 +2972,7 @@ fn short_stability(item: &clean::Item, cx: &Context) -> Vec<String> {
             message = format!(
                 "<details><summary>{}</summary>{}</details>",
                 message,
-                MarkdownHtml(&unstable_reason, RefCell::new(&mut ids), error_codes)
+                MarkdownHtml(&unstable_reason, RefCell::new(&mut ids), error_codes, cx.edition)
             );
         }
 
@@ -4197,7 +4202,8 @@ fn render_impl(w: &mut fmt::Formatter<'_>, cx: &Context, i: &Impl, link: AssocIt
         if let Some(ref dox) = cx.shared.maybe_collapsed_doc_value(&i.impl_item) {
             let mut ids = cx.id_map.borrow_mut();
             write!(w, "<div class='docblock'>{}</div>",
-                   Markdown(&*dox, &i.impl_item.links(), RefCell::new(&mut ids), cx.codes))?;
+                   Markdown(&*dox, &i.impl_item.links(), RefCell::new(&mut ids),
+                            cx.codes, cx.edition))?;
         }
     }