XXX: NtExpr/NtLiteral

Notes about tests:
- tests/ui/macros/stringify.rs: the `c2` macro is no longer needed,
  because the TokenStream pretty printer is now used for all cases.

- tests/ui/rfcs/rfc-2294-if-let-guard/feature-gate.rs: some messages are
  now duplicated due to repeated parsing.

- tests/ui/rfcs/rfc-2497-if-let-chains/disallowed-positions*.rs: ditto.

XXX: Getting a test failure here:
```
Building tool error_index_generator (stage1 -> stage2, x86_64-unknown-linux-gnu)
   Compiling cfg-if v1.0.0
    ...
   Compiling mdbook v0.4.37
error: internal compiler error: the following error was constructed but not emitted

error: unexpected token: keyword `self`
   --> /home/njn/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mdbook-0.4.37/src/book/summary.rs:280:31
    |
280 | ...   bail!(self.parse_error("Suffix chapters cannot be followed by a list"));
    |             ^^^^

thread 'rustc' panicked at /home/njn/dev/rust3/compiler/rustc_errors/src/diagnostic.rs:1375:17:
error was constructed but not emitted
```
I get a similar compile error (not ICE) in a vanilla compiler if I change `can_begin_maybe_minus` to accept `NtExpr` without checking that `e` is a `Lit` or `Unary`

error: format argument must be a string literal
   --> /home/njn/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mdbook-0.4.37/src/book/summary.rs:280:31
    |
280 | ...   bail!(self.parse_error("Suffix chapters cannot be followed by a list"));
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
help: you might be missing a string literal to format with
    |
280 |                         bail!("{}", self.parse_error("Suffix chapters cannot be followed by a list"));
    |                               +++++

Author: nnethercote

compiler/rustc_ast/src/ast_traits.rs

@@ -231,13 +231,11 @@ impl HasTokens for Attribute {
 impl HasTokens for Nonterminal {
     fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
             Nonterminal::NtBlock(block) => block.tokens(),
         }
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),
         }
     }

compiler/rustc_ast/src/mut_visit.rs

@@ -820,8 +820,6 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
 fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
     match nt {
         token::NtBlock(block) => vis.visit_block(block),
-        token::NtExpr(expr) => vis.visit_expr(expr),
-        token::NtLiteral(expr) => vis.visit_expr(expr),
     }
 }
 

compiler/rustc_ast/src/token.rs

@@ -134,16 +134,27 @@ impl Lit {
         }
     }
 
-    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` excluding unary negation.
+    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
+    /// `Parser::maybe_parse_token_lit` (excluding unary negation).
     pub fn from_token(token: &Token) -> Option<Lit> {
         match token.uninterpolate().kind {
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
             Literal(token_lit) => Some(token_lit),
-            Interpolated(ref nt)
-                if let NtExpr(expr) | NtLiteral(expr) = &**nt
-                    && let ast::ExprKind::Lit(token_lit) = expr.kind =>
-            {
-                Some(token_lit)
+            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Literal))) => {
+                panic!("njn: FROM_TOKEN (1)");
+                // if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                // && let ast::ExprKind::Lit(token_lit) = expr.kind =>
+                // {
+                //     Some(token_lit)
+                // }
+            }
+            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Expr))) => {
+                panic!("njn: FROM_TOKEN (2)");
+                // if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                // && let ast::ExprKind::Lit(token_lit) = expr.kind =>
+                // {
+                //     Some(token_lit)
+                // }
             }
             _ => None,
         }
@@ -476,6 +487,7 @@ impl Token {
         Token::new(Ident(ident.name, ident.is_raw_guess().into()), ident.span)
     }
 
+    /// njn: phase this out in favour of Parser::uninterpolated_token_span?
     /// For interpolated tokens, returns a span of the fragment to which the interpolated
     /// token refers. For all other tokens this is just a regular span.
     /// It is particularly important to use this for identifiers and lifetimes
@@ -531,9 +543,7 @@ impl Token {
             PathSep                           | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
-                NtExpr(..)    |
-                NtBlock(..)),
+            Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Block |
                 NonterminalKind::Expr |
@@ -561,7 +571,7 @@ impl Token {
             | DotDot | DotDotDot | DotDotEq      // ranges
             | Lt | BinOp(Shl)                    // associated path
             | PathSep => true,                   // global path
-            Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) | NtBlock(..)),
+            Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Block |
                 NonterminalKind::PatParam { .. } |
@@ -603,6 +613,7 @@ impl Token {
         match self.kind {
             OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
+            Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Expr
                 | NonterminalKind::Expr2021 { .. }
@@ -649,22 +660,24 @@ impl Token {
     ///
     /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
     ///
-    /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
+    /// Keep this in sync with `Lit::from_token` and
+    /// `Parser::maybe_parse_token_lit` (excluding unary negation).
     pub fn can_begin_literal_maybe_minus(&self) -> bool {
         match self.uninterpolate().kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
-            Interpolated(ref nt) => match &**nt {
-                NtLiteral(_) => true,
-                NtExpr(e) => match &e.kind {
-                    ast::ExprKind::Lit(_) => true,
-                    ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
-                        matches!(&e.kind, ast::ExprKind::Lit(_))
-                    }
-                    _ => false,
-                },
-                _ => false,
-            },
+            // njn: fix up
+            // Interpolated(ref nt) => match &**nt {
+            //     NtLiteral(_) => true,
+            //     NtExpr(e) => match &e.kind {
+            //         ast::ExprKind::Lit(_) => true,
+            //         ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
+            //             matches!(&e.kind, ast::ExprKind::Lit(_))
+            //         }
+            //         _ => false,
+            //     },
+            //     _ => false,
+            // },
             // njn: too simple compared to what's above?
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Literal | NonterminalKind::Expr | NonterminalKind::Expr2021 { .. },
@@ -676,14 +689,19 @@ impl Token {
     pub fn can_begin_string_literal(&self) -> bool {
         match self.uninterpolate().kind {
             Literal(..) => true,
-            Interpolated(ref nt) => match &**nt {
-                NtLiteral(_) => true,
-                NtExpr(e) => match &e.kind {
-                    ast::ExprKind::Lit(_) => true,
-                    _ => false,
-                },
-                _ => false,
-            },
+            // njn: fix up
+            // Interpolated(ref nt) => match &**nt {
+            //     NtLiteral(_) => true,
+            //     NtExpr(e) => match &e.kind {
+            //         ast::ExprKind::Lit(_) => true,
+            //         _ => false,
+            //     },
+            //     _ => false,
+            // },
+            // njn: too simple compared to what's above?
+            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+                NonterminalKind::Literal | NonterminalKind::Expr | NonterminalKind::Expr2021,
+            ))) => true,
             _ => false,
         }
     }
@@ -738,16 +756,19 @@ impl Token {
         self.ident().is_some_and(|(ident, _)| ident.name == name)
     }
 
-    /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
+    /// Would `maybe_reparse_metavar_expr` in `parser.rs` return `Ok(..)`?
     /// That is, is this a pre-parsed expression dropped into the token stream
     /// (which happens while parsing the result of macro expansion)?
-    pub fn is_whole_expr(&self) -> bool {
-        #[allow(irrefutable_let_patterns)] // njn: temp
+    pub fn is_metavar_expr(&self) -> bool {
+        #[allow(irrefutable_let_patterns)] // FIXME: temporary
         if let Interpolated(nt) = &self.kind
-            && let NtExpr(_) | NtLiteral(_) = &**nt
+            && let NtBlock(_) = &**nt
         {
             true
-        } else if matches!(self.is_metavar_seq(), Some(NonterminalKind::Path)) {
+        } else if matches!(
+            self.is_metavar_seq(),
+            Some(NonterminalKind::Expr | NonterminalKind::Literal | NonterminalKind::Path)
+        ) {
             true
         } else {
             false
@@ -756,6 +777,7 @@ impl Token {
 
     /// Is the token an interpolated block (`$b:block`)?
     pub fn is_whole_block(&self) -> bool {
+        #[allow(irrefutable_let_patterns)] // FIXME: temporary
         if let Interpolated(nt) = &self.kind
             && let NtBlock(..) = &**nt
         {
@@ -928,8 +950,6 @@ impl PartialEq<TokenKind> for Token {
 /// For interpolation during macro expansion.
 pub enum Nonterminal {
     NtBlock(P<ast::Block>),
-    NtExpr(P<ast::Expr>),
-    NtLiteral(P<ast::Expr>),
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -1027,15 +1047,12 @@ impl Nonterminal {
     pub fn use_span(&self) -> Span {
         match self {
             NtBlock(block) => block.span,
-            NtExpr(expr) | NtLiteral(expr) => expr.span,
         }
     }
 
     pub fn descr(&self) -> &'static str {
         match self {
             NtBlock(..) => "block",
-            NtExpr(..) => "expression",
-            NtLiteral(..) => "literal",
         }
     }
 }
@@ -1054,8 +1071,6 @@ impl fmt::Debug for Nonterminal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
             NtBlock(..) => f.pad("NtBlock(..)"),
-            NtExpr(..) => f.pad("NtExpr(..)"),
-            NtLiteral(..) => f.pad("NtLiteral(..)"),
         }
     }
 }
@@ -1077,7 +1092,7 @@ mod size_asserts {
     // tidy-alphabetical-start
     static_assert_size!(Lit, 12);
     static_assert_size!(LitKind, 2);
-    static_assert_size!(Nonterminal, 16);
+    static_assert_size!(Nonterminal, 8);
     static_assert_size!(Token, 24);
     static_assert_size!(TokenKind, 16);
     // tidy-alphabetical-end

compiler/rustc_ast/src/tokenstream.rs

@@ -467,7 +467,6 @@ impl TokenStream {
     pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
         match nt {
             Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
         }
     }
 

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -316,6 +316,12 @@ pub(super) fn transcribe<'a>(
                         MatchedSingle(ParseNtResult::PatWithOr(ref pat)) => {
                             mk_delimited(NonterminalKind::PatWithOr, TokenStream::from_ast(pat))
                         }
+                        MatchedSingle(ParseNtResult::Expr(ref expr)) => {
+                            mk_delimited(NonterminalKind::Expr, TokenStream::from_ast(expr))
+                        }
+                        MatchedSingle(ParseNtResult::Literal(ref expr)) => {
+                            mk_delimited(NonterminalKind::Literal, TokenStream::from_ast(expr))
+                        }
                         MatchedSingle(ParseNtResult::Ty(ref ty)) => {
                             mk_delimited(NonterminalKind::Ty, TokenStream::from_ast(ty))
                         }

compiler/rustc_index/src/bit_set/tests.rs

@@ -302,7 +302,7 @@ fn chunked_bitset() {
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x8000_0000_0000_0000
             ])),
-        ],
+        ], // njn: trailing comma here causes a crash in reparse_metavar_seq?
     );
     assert_eq!(b4096.count(), 2);
     b4096.assert_valid();
@@ -336,7 +336,7 @@ fn chunked_bitset() {
             ])),
             Zeros(2048),
             Zeros(1808),
-        ],
+        ], // njn: trailing comma here causes a crash in reparse_metavar_seq?
     );
     let mut b10000b = ChunkedBitSet::<usize>::new_empty(10000);
     b10000b.clone_from(&b10000);

compiler/rustc_parse/messages.ftl

@@ -807,7 +807,7 @@ parse_unexpected_parentheses_in_match_arm_pattern = unexpected parentheses surro
 parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
     .note = you cannot use `Self` as a generic parameter because it is reserved for associated items
 
-parse_unexpected_token_after_dot = unexpected token: `{$actual}`
+parse_unexpected_token_after_dot = unexpected token: {$actual}
 
 parse_unexpected_token_after_label = expected `while`, `for`, `loop` or `{"{"}` after a label
     .suggestion_remove_label = consider removing the label

compiler/rustc_parse/src/errors.rs

@@ -1602,10 +1602,10 @@ pub(crate) struct SelfArgumentPointer {
 
 #[derive(Diagnostic)]
 #[diag(parse_unexpected_token_after_dot)]
-pub struct UnexpectedTokenAfterDot<'a> {
+pub struct UnexpectedTokenAfterDot {
     #[primary_span]
     pub span: Span,
-    pub actual: Cow<'a, str>,
+    pub actual: String,
 }
 
 #[derive(Diagnostic)]