Collect tokens when handling :literal matcher

An `NtLiteral` just wraps an `Expr`, so we don't need to add a new `tokens`
field to an AST struct.

Author: Aaron1011

compiler/rustc_parse/src/lib.rs

@@ -278,7 +278,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
             Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
         }
         Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
-        Nonterminal::NtExpr(ref expr) => {
+        Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
             if expr.tokens.is_none() {
                 debug!("missing tokens for expr {:?}", expr);
             }

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -140,7 +140,15 @@ impl<'a> Parser<'a> {
                 }
                 token::NtExpr(expr)
             }
-            NonterminalKind::Literal => token::NtLiteral(self.parse_literal_maybe_minus()?),
+            NonterminalKind::Literal => {
+                let (mut lit, tokens) =
+                    self.collect_tokens(|this| this.parse_literal_maybe_minus())?;
+                // We have have eaten a nonterminal, which  could already have tokens
+                if lit.tokens.is_none() {
+                    lit.tokens = Some(tokens);
+                }
+                token::NtLiteral(lit)
+            }
             NonterminalKind::Ty => {
                 let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?;
                 // We have an eaten an NtTy, which could already have tokens