rustc_parse: Better spans for synthesized token streams

Author: petrochenkov

compiler/rustc_ast/src/token.rs

@@ -771,7 +771,7 @@ impl fmt::Display for NonterminalKind {
 }
 
 impl Nonterminal {
-    fn span(&self) -> Span {
+    pub fn span(&self) -> Span {
         match self {
             NtItem(item) => item.span,
             NtBlock(block) => block.span,

compiler/rustc_ast_lowering/src/lib.rs

@@ -206,8 +206,7 @@ pub trait ResolverAstLowering {
     ) -> LocalDefId;
 }
 
-type NtToTokenstream =
-    fn(&Nonterminal, &ParseSess, Span, CanSynthesizeMissingTokens) -> TokenStream;
+type NtToTokenstream = fn(&Nonterminal, &ParseSess, CanSynthesizeMissingTokens) -> TokenStream;
 
 /// Context of `impl Trait` in code, which determines whether it is allowed in an HIR subtree,
 /// and if so, what meaning it has.
@@ -417,12 +416,7 @@ impl<'a> TokenStreamLowering<'a> {
     fn lower_token(&mut self, token: Token) -> TokenStream {
         match token.kind {
             token::Interpolated(nt) => {
-                let tts = (self.nt_to_tokenstream)(
-                    &nt,
-                    self.parse_sess,
-                    token.span,
-                    self.synthesize_tokens,
-                );
+                let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
                 TokenTree::Delimited(
                     DelimSpan::from_single(token.span),
                     DelimToken::NoDelim,

compiler/rustc_expand/src/base.rs

@@ -141,7 +141,7 @@ impl Annotatable {
     }
 
     crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
-        nt_to_tokenstream(&self.into_nonterminal(), sess, DUMMY_SP, CanSynthesizeMissingTokens::No)
+        nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::No)
     }
 
     pub fn expect_item(self) -> P<ast::Item> {

compiler/rustc_expand/src/expand.rs

@@ -743,7 +743,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                         AttrStyle::Inner => rustc_parse::fake_token_stream(
                             &self.cx.sess.parse_sess,
                             &item.into_nonterminal(),
-                            span,
                         ),
                     };
                     let attr_item = attr.unwrap_normal_item();

compiler/rustc_expand/src/proc_macro.rs

@@ -94,12 +94,7 @@ impl MultiItemModifier for ProcMacroDerive {
         let input = if item.pretty_printing_compatibility_hack() {
             TokenTree::token(token::Interpolated(Lrc::new(item)), DUMMY_SP).into()
         } else {
-            nt_to_tokenstream(
-                &item,
-                &ecx.sess.parse_sess,
-                DUMMY_SP,
-                CanSynthesizeMissingTokens::Yes,
-            )
+            nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::Yes)
         };
 
         let server = proc_macro_server::Rustc::new(ecx);

compiler/rustc_expand/src/proc_macro_server.rs

@@ -179,7 +179,7 @@ impl FromInternal<(TreeAndSpacing, &'_ ParseSess, &'_ mut Vec<Self>)>
                 {
                     TokenTree::Ident(Ident::new(sess, name.name, is_raw, name.span))
                 } else {
-                    let stream = nt_to_tokenstream(&nt, sess, span, CanSynthesizeMissingTokens::No);
+                    let stream = nt_to_tokenstream(&nt, sess, CanSynthesizeMissingTokens::No);
                     TokenTree::Group(Group {
                         delimiter: Delimiter::None,
                         stream,

compiler/rustc_parse/src/lib.rs

@@ -236,7 +236,6 @@ pub fn parse_in<'a, T>(
 pub fn nt_to_tokenstream(
     nt: &Nonterminal,
     sess: &ParseSess,
-    span: Span,
     synthesize_tokens: CanSynthesizeMissingTokens,
 ) -> TokenStream {
     // A `Nonterminal` is often a parsed AST item. At this point we now
@@ -256,11 +255,9 @@ pub fn nt_to_tokenstream(
         |tokens: Option<&LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream());
 
     let tokens = match *nt {
-        Nonterminal::NtItem(ref item) => {
-            prepend_attrs(sess, &item.attrs, nt, span, item.tokens.as_ref())
-        }
+        Nonterminal::NtItem(ref item) => prepend_attrs(sess, &item.attrs, nt, item.tokens.as_ref()),
         Nonterminal::NtBlock(ref block) => convert_tokens(block.tokens.as_ref()),
-        Nonterminal::NtStmt(ref stmt) => prepend_attrs(sess, stmt.attrs(), nt, span, stmt.tokens()),
+        Nonterminal::NtStmt(ref stmt) => prepend_attrs(sess, stmt.attrs(), nt, stmt.tokens()),
         Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()),
         Nonterminal::NtTy(ref ty) => convert_tokens(ty.tokens.as_ref()),
         Nonterminal::NtIdent(ident, is_raw) => {
@@ -277,31 +274,30 @@ pub fn nt_to_tokenstream(
             if expr.tokens.is_none() {
                 debug!("missing tokens for expr {:?}", expr);
             }
-            prepend_attrs(sess, &expr.attrs, nt, span, expr.tokens.as_ref())
+            prepend_attrs(sess, &expr.attrs, nt, expr.tokens.as_ref())
         }
     };
 
     if let Some(tokens) = tokens {
         return tokens;
     } else if matches!(synthesize_tokens, CanSynthesizeMissingTokens::Yes) {
-        return fake_token_stream(sess, nt, span);
+        return fake_token_stream(sess, nt);
     } else {
         let pretty = rustc_ast_pretty::pprust::nonterminal_to_string_no_extra_parens(&nt);
-        panic!("Missing tokens at {:?} for nt {:?}", span, pretty);
+        panic!("Missing tokens for nt {:?}", pretty);
     }
 }
 
-pub fn fake_token_stream(sess: &ParseSess, nt: &Nonterminal, span: Span) -> TokenStream {
+pub fn fake_token_stream(sess: &ParseSess, nt: &Nonterminal) -> TokenStream {
     let source = pprust::nonterminal_to_string(nt);
     let filename = FileName::macro_expansion_source_code(&source);
-    parse_stream_from_source_str(filename, source, sess, Some(span))
+    parse_stream_from_source_str(filename, source, sess, Some(nt.span()))
 }
 
 fn prepend_attrs(
     sess: &ParseSess,
     attrs: &[ast::Attribute],
     nt: &Nonterminal,
-    span: Span,
     tokens: Option<&tokenstream::LazyTokenStream>,
 ) -> Option<tokenstream::TokenStream> {
     if attrs.is_empty() {
@@ -312,7 +308,7 @@ fn prepend_attrs(
         // FIXME: Correctly handle tokens for inner attributes.
         // For now, we fall back to reparsing the original AST node
         if attr.style == ast::AttrStyle::Inner {
-            return Some(fake_token_stream(sess, nt, span));
+            return Some(fake_token_stream(sess, nt));
         }
         builder.push(attr.tokens());
     }