parser: Set previous and unnormalized tokens in couple more places

Author: petrochenkov

src/librustc_parse/lib.rs

@@ -172,6 +172,7 @@ fn maybe_source_file_to_parser(
     parser.unclosed_delims = unclosed_delims;
     if parser.token == token::Eof && parser.token.span.is_dummy() {
         parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
+        assert!(parser.unnormalized_token.is_none());
     }
 
     Ok(parser)

src/librustc_parse/parser/item.rs

@@ -1400,8 +1400,9 @@ impl<'a> Parser<'a> {
     }
 
     fn report_invalid_macro_expansion_item(&self, args: &MacArgs) {
+        let span = args.span().expect("undelimited macro call");
         let mut err = self.struct_span_err(
-            self.prev_span,
+            span,
             "macros that expand to items must be delimited with braces or followed by a semicolon",
         );
         if self.unclosed_delims.is_empty() {
@@ -1416,14 +1417,14 @@ impl<'a> Parser<'a> {
             );
         } else {
             err.span_suggestion(
-                self.prev_span,
+                span,
                 "change the delimiters to curly braces",
                 " { /* items */ }".to_string(),
                 Applicability::HasPlaceholders,
             );
         }
         err.span_suggestion(
-            self.prev_span.shrink_to_hi(),
+            span.shrink_to_hi(),
             "add a semicolon",
             ';'.to_string(),
             Applicability::MaybeIncorrect,

src/librustc_parse/parser/mod.rs

@@ -95,7 +95,7 @@ pub struct Parser<'a> {
     /// The current non-normalized token if it's different from `token`.
     /// Preferable use is through the `unnormalized_token()` getter.
     /// Use span from this token if you need to concatenate it with some neighbouring spans.
-    unnormalized_token: Option<Token>,
+    pub unnormalized_token: Option<Token>,
     /// The previous normalized token.
     /// Use span from this token if you need an isolated span.
     prev_token: Token,
@@ -1096,15 +1096,15 @@ impl<'a> Parser<'a> {
                     &mut self.token_cursor.frame,
                     self.token_cursor.stack.pop().unwrap(),
                 );
-                self.token.span = frame.span.entire();
+                self.token = Token::new(TokenKind::CloseDelim(frame.delim), frame.span.close);
+                self.unnormalized_token = None;
                 self.bump();
                 TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream.into())
             }
             token::CloseDelim(_) | token::Eof => unreachable!(),
             _ => {
-                let token = self.token.clone();
                 self.bump();
-                TokenTree::Token(token)
+                TokenTree::Token(self.prev_token.clone())
             }
         }
     }