XXX: NtExpr/NtLiteral

[xt] move from_token

Getting a test failure here:
```
Building tool error_index_generator (stage1 -> stage2, x86_64-unknown-linux-gnu)
   Compiling cfg-if v1.0.0
    ...
   Compiling mdbook v0.4.31
error: internal compiler error: the following error was constructed but not emitted

error: unexpected token: keyword `self`
   --> /home/njn/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mdbook-0.4.31/src/book/summary.rs:275:31
    |
275 |                         bail!(self.parse_error("Suffix chapters cannot be followed by a list"));
    |                               ^^^^

thread 'rustc' panicked at compiler/rustc_errors/src/diagnostic_builder.rs:775:21:
error was constructed but not emitted
```

Author: nnethercote

compiler/rustc_ast/src/ast_traits.rs

@@ -231,13 +231,11 @@ impl HasTokens for Attribute {
 impl HasTokens for Nonterminal {
     fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
             Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
             Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }

compiler/rustc_ast/src/mut_visit.rs

@@ -789,10 +789,8 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
 // multiple items there....
 pub fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
     match nt {
-        token::NtExpr(expr) => vis.visit_expr(expr),
         token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
         token::NtLifetime(ident) => vis.visit_ident(ident),
-        token::NtLiteral(expr) => vis.visit_expr(expr),
     }
 }
 

compiler/rustc_ast/src/token.rs

@@ -4,7 +4,6 @@ pub use Nonterminal::*;
 pub use TokenKind::*;
 
 use crate::ast;
-use crate::ptr::P;
 use crate::util::case::Case;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -134,18 +133,27 @@ impl Lit {
         }
     }
 
-    /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
+    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
+    /// `Parser::maybe_parse_token_lit` (excluding unary negation).
     pub fn from_token(token: &Token) -> Option<Lit> {
         match token.uninterpolate().kind {
-            Ident(name, false) if name.is_bool_lit() => {
-                Some(Lit::new(Bool, name, None))
-            }
+            Ident(name, false) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
             Literal(token_lit) => Some(token_lit),
-            Interpolated(ref nt)
-                if let NtExpr(expr) | NtLiteral(expr) = &**nt
-                && let ast::ExprKind::Lit(token_lit) = expr.kind =>
-            {
-                Some(token_lit)
+            OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Literal))) => {
+                panic!("njn: FROM_TOKEN (1)");
+                // if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                // && let ast::ExprKind::Lit(token_lit) = expr.kind =>
+                // {
+                //     Some(token_lit)
+                // }
+            }
+            OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Expr))) => {
+                panic!("njn: FROM_TOKEN (2)");
+                // if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                // && let ast::ExprKind::Lit(token_lit) = expr.kind =>
+                // {
+                //     Some(token_lit)
+                // }
             }
             _ => None,
         }
@@ -417,6 +425,7 @@ impl Token {
         Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
     }
 
+    /// njn: phase this out in favour of Parser::uninterpolated_span
     /// For interpolated tokens, returns a span of the fragment to which the interpolated
     /// token refers. For all other tokens this is just a regular span.
     /// It is particularly important to use this for identifiers and lifetimes
@@ -469,9 +478,11 @@ impl Token {
             ModSep                            | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) => matches!(**nt, NtLiteral(..) | NtExpr(..)),
             OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
-                NonterminalKind::Block | NonterminalKind::Path
+                NonterminalKind::Block |
+                NonterminalKind::Expr |
+                NonterminalKind::Literal |
+                NonterminalKind::Path
             )))
                 => true,
             _ => false,
@@ -494,12 +505,12 @@ impl Token {
             | DotDot | DotDotDot | DotDotEq      // ranges
             | Lt | BinOp(Shl)                    // associated path
             | ModSep => true,                    // global path
-            Interpolated(ref nt) => matches!(**nt, NtLiteral(..)),
             | OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
                 NonterminalKind::Block |
                 NonterminalKind::PatParam { .. } |
                 NonterminalKind::PatWithOr |
-                NonterminalKind::Path
+                NonterminalKind::Path |
+                NonterminalKind::Literal
             ))) => true,
             _ => false,
         }
@@ -532,10 +543,9 @@ impl Token {
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
             OpenDelim(Delimiter::Brace) => true,
-            Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtLiteral(..)),
-            OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(NonterminalKind::Block))) => {
-                true
-            }
+            OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
+                NonterminalKind::Block | NonterminalKind::Expr | NonterminalKind::Literal,
+            ))) => true,
             _ => self.can_begin_literal_maybe_minus(),
         }
     }
@@ -584,22 +594,15 @@ impl Token {
     ///
     /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
     ///
-    /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
+    /// Keep this in sync with `Lit::from_token` and
+    /// `Parser::maybe_parse_token_lit` (excluding unary negation).
     pub fn can_begin_literal_maybe_minus(&self) -> bool {
         match self.uninterpolate().kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,
-            Interpolated(ref nt) => match &**nt {
-                NtLiteral(_) => true,
-                NtExpr(e) => match &e.kind {
-                    ast::ExprKind::Lit(_) => true,
-                    ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
-                        matches!(&e.kind, ast::ExprKind::Lit(_))
-                    }
-                    _ => false,
-                },
-                _ => false,
-            },
+            OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(
+                NonterminalKind::Literal | NonterminalKind::Expr,
+            ))) => true,
             _ => false,
         }
     }
@@ -615,7 +618,6 @@ impl Token {
                     Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
                 }
                 NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
-                _ => Cow::Borrowed(self),
             },
             _ => Cow::Borrowed(self),
         }
@@ -665,22 +667,19 @@ impl Token {
         self.ident().is_some_and(|(ident, _)| ident.name == name)
     }
 
-    /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
+    /// Would `maybe_reparse_metavar_expr` in `parser.rs` return `Ok(..)`?
     /// That is, is this a pre-parsed expression dropped into the token stream
     /// (which happens while parsing the result of macro expansion)?
-    pub fn is_whole_expr(&self) -> bool {
-        if let Interpolated(nt) = &self.kind
-            && let NtExpr(_) | NtLiteral(_) = **nt
-        {
-            true
-        } else if matches!(
+    pub fn is_metavar_expr(&self) -> bool {
+        matches!(
             self.is_metavar_seq(),
-            Some(NonterminalKind::Block | NonterminalKind::Path)
-        ) {
-            true
-        } else {
-            false
-        }
+            Some(
+                NonterminalKind::Expr
+                    | NonterminalKind::Literal
+                    | NonterminalKind::Block
+                    | NonterminalKind::Path
+            )
+        )
     }
 
     /// Are we at a block from a metavar (`$b:block`)?
@@ -843,10 +842,8 @@ impl PartialEq<TokenKind> for Token {
 #[derive(Clone, Encodable, Decodable)]
 /// For interpolation during macro expansion.
 pub enum Nonterminal {
-    NtExpr(P<ast::Expr>),
     NtIdent(Ident, /* is_raw */ bool),
     NtLifetime(Ident),
-    NtLiteral(P<ast::Expr>),
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -930,7 +927,6 @@ impl fmt::Display for NonterminalKind {
 impl Nonterminal {
     pub fn span(&self) -> Span {
         match self {
-            NtExpr(expr) | NtLiteral(expr) => expr.span,
             NtIdent(ident, _) | NtLifetime(ident) => ident.span,
         }
     }
@@ -955,9 +951,7 @@ impl PartialEq for Nonterminal {
 impl fmt::Debug for Nonterminal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            NtExpr(..) => f.pad("NtExpr(..)"),
             NtIdent(..) => f.pad("NtIdent(..)"),
-            NtLiteral(..) => f.pad("NtLiteral(..)"),
             NtLifetime(..) => f.pad("NtLifetime(..)"),
         }
     }

compiler/rustc_ast/src/tokenstream.rs

@@ -444,6 +444,7 @@ impl TokenStream {
         let Some(tokens) = node.tokens() else {
             panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
         };
+        //eprintln!("from_ast: {tokens:#?}");
         let attrs = node.attrs();
         let attr_stream = if attrs.is_empty() {
             tokens.to_attr_token_stream()
@@ -463,7 +464,6 @@ impl TokenStream {
             Nonterminal::NtLifetime(ident) => {
                 TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
             }
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
         }
     }
 

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -726,10 +726,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
 
     fn nonterminal_to_string(&self, nt: &Nonterminal) -> String {
         match nt {
-            token::NtExpr(e) => self.expr_to_string(e),
             token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(*e, *is_raw).to_string(),
             token::NtLifetime(e) => e.to_string(),
-            token::NtLiteral(e) => self.expr_to_string(e),
         }
     }
 

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -221,6 +221,9 @@ pub(super) fn transcribe<'a>(
                         // Emit as a token stream within `Delimiter::Invisible` to maintain parsing
                         // priorities.
                         marker.visit_span(&mut sp);
+                        // njn: `sp` usage here means that both the open delim
+                        // and close delim end up with the same span, which
+                        // covers the `$foo` in the decl macro RHS
                         TokenTree::Delimited(
                             DelimSpan::from_single(sp),
                             Delimiter::Invisible(InvisibleSource::MetaVar(nt_kind)),
@@ -255,6 +258,12 @@ pub(super) fn transcribe<'a>(
                         MatchedSingle(ParseNtResult::PatWithOr(ref pat)) => {
                             mk_delimited(NonterminalKind::PatWithOr, TokenStream::from_ast(pat))
                         }
+                        MatchedSingle(ParseNtResult::Expr(ref expr)) => {
+                            mk_delimited(NonterminalKind::Expr, TokenStream::from_ast(expr))
+                        }
+                        MatchedSingle(ParseNtResult::Literal(ref expr)) => {
+                            mk_delimited(NonterminalKind::Literal, TokenStream::from_ast(expr))
+                        }
                         MatchedSingle(ParseNtResult::Ty(ref ty)) => {
                             mk_delimited(NonterminalKind::Ty, TokenStream::from_ast(ty))
                         }

compiler/rustc_index/src/bit_set/tests.rs

@@ -302,7 +302,7 @@ fn chunked_bitset() {
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x8000_0000_0000_0000
             ])),
-        ],
+        ] // njn: trailing comma here causes a crash in reparse_metavar_seq
     );
     assert_eq!(b4096.count(), 2);
     b4096.assert_valid();
@@ -336,7 +336,7 @@ fn chunked_bitset() {
             ])),
             Zeros(2048),
             Zeros(1808),
-        ],
+        ]
     );
     let mut b10000b = ChunkedBitSet::<usize>::new_empty(10000);
     b10000b.clone_from(&b10000);