Auto merge of #138478 - nnethercote:rm-NtExpr-NtLiteral, r=petrochenkov

Remove `NtExpr` and `NtLiteral`

The next part of #124141.

r? `@petrochenkov`

Author: bors

compiler/rustc_ast/src/ast_traits.rs

@@ -209,13 +209,11 @@ impl HasTokens for Attribute {
 impl HasTokens for Nonterminal {
     fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
             Nonterminal::NtBlock(block) => block.tokens(),
         }
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),
         }
     }

compiler/rustc_ast/src/mut_visit.rs

@@ -899,8 +899,6 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
 fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
     match nt {
         token::NtBlock(block) => vis.visit_block(block),
-        token::NtExpr(expr) => vis.visit_expr(expr),
-        token::NtLiteral(expr) => vis.visit_expr(expr),
     }
 }
 

compiler/rustc_ast/src/token.rs

@@ -198,16 +198,17 @@ impl Lit {
         }
     }
 
-    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` excluding unary negation.
+    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
+    /// `Parser::eat_token_lit` (excluding unary negation).
     pub fn from_token(token: &Token) -> Option<Lit> {
         match token.uninterpolate().kind {
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
             Literal(token_lit) => Some(token_lit),
-            Interpolated(ref nt)
-                if let NtExpr(expr) | NtLiteral(expr) = &**nt
-                    && let ast::ExprKind::Lit(token_lit) = expr.kind =>
-            {
-                Some(token_lit)
+            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+                MetaVarKind::Literal | MetaVarKind::Expr { .. },
+            ))) => {
+                // Unreachable with the current test suite.
+                panic!("from_token metavar");
             }
             _ => None,
         }
@@ -447,8 +448,9 @@ pub enum TokenKind {
 
     /// Identifier token.
     /// Do not forget about `NtIdent` when you want to match on identifiers.
-    /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
-    /// treat regular and interpolated identifiers in the same way.
+    /// It's recommended to use `Token::{ident,uninterpolate}` and
+    /// `Parser::token_uninterpolated_span` to treat regular and interpolated
+    /// identifiers in the same way.
     Ident(Symbol, IdentIsRaw),
     /// This identifier (and its span) is the identifier passed to the
     /// declarative macro. The span in the surrounding `Token` is the span of
@@ -457,8 +459,9 @@ pub enum TokenKind {
 
     /// Lifetime identifier token.
     /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
-    /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
-    /// treat regular and interpolated lifetime identifiers in the same way.
+    /// It's recommended to use `Token::{ident,uninterpolate}` and
+    /// `Parser::token_uninterpolated_span` to treat regular and interpolated
+    /// identifiers in the same way.
     Lifetime(Symbol, IdentIsRaw),
     /// This identifier (and its span) is the lifetime passed to the
     /// declarative macro. The span in the surrounding `Token` is the span of
@@ -584,20 +587,6 @@ impl Token {
         Token::new(Ident(ident.name, ident.is_raw_guess().into()), ident.span)
     }
 
-    /// For interpolated tokens, returns a span of the fragment to which the interpolated
-    /// token refers. For all other tokens this is just a regular span.
-    /// It is particularly important to use this for identifiers and lifetimes
-    /// for which spans affect name resolution and edition checks.
-    /// Note that keywords are also identifiers, so they should use this
-    /// if they keep spans or perform edition checks.
-    pub fn uninterpolated_span(&self) -> Span {
-        match self.kind {
-            NtIdent(ident, _) | NtLifetime(ident, _) => ident.span,
-            Interpolated(ref nt) => nt.use_span(),
-            _ => self.span,
-        }
-    }
-
     pub fn is_range_separator(&self) -> bool {
         [DotDot, DotDotDot, DotDotEq].contains(&self.kind)
     }
@@ -642,12 +631,7 @@ impl Token {
             PathSep                           | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) =>
-                matches!(&**nt,
-                    NtBlock(..)   |
-                    NtExpr(..)    |
-                    NtLiteral(..)
-                ),
+            Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Block |
                 MetaVarKind::Expr { .. } |
@@ -677,11 +661,6 @@ impl Token {
             Lt |                                 // path (UFCS constant)
             Shl => true,                         // path (double UFCS)
             Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern
-            Interpolated(nt) =>
-                matches!(&**nt,
-                    | NtExpr(..)
-                    | NtLiteral(..)
-                ),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Expr { .. } |
                 MetaVarKind::Literal |
@@ -724,7 +703,7 @@ impl Token {
         match self.kind {
             OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
-            Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
+            Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
             ))) => true,
@@ -768,22 +747,12 @@ impl Token {
     ///
     /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
     ///
-    /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
+    /// Keep this in sync with `Lit::from_token` and `Parser::eat_token_lit`
+    /// (excluding unary negation).
     pub fn can_begin_literal_maybe_minus(&self) -> bool {
         match self.uninterpolate().kind {
             Literal(..) | Minus => true,
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
-            Interpolated(ref nt) => match &**nt {
-                NtLiteral(_) => true,
-                NtExpr(e) => match &e.kind {
-                    ast::ExprKind::Lit(_) => true,
-                    ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
-                        matches!(&e.kind, ast::ExprKind::Lit(_))
-                    }
-                    _ => false,
-                },
-                _ => false,
-            },
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
                 MetaVarKind::Literal => true,
                 MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
@@ -798,14 +767,6 @@ impl Token {
     pub fn can_begin_string_literal(&self) -> bool {
         match self.uninterpolate().kind {
             Literal(..) => true,
-            Interpolated(ref nt) => match &**nt {
-                NtLiteral(_) => true,
-                NtExpr(e) => match &e.kind {
-                    ast::ExprKind::Lit(_) => true,
-                    _ => false,
-                },
-                _ => false,
-            },
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
                 MetaVarKind::Literal => true,
                 MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
@@ -869,19 +830,25 @@ impl Token {
 
     /// Is this a pre-parsed expression dropped into the token stream
     /// (which happens while parsing the result of macro expansion)?
-    pub fn is_whole_expr(&self) -> bool {
+    pub fn is_metavar_expr(&self) -> bool {
         #[allow(irrefutable_let_patterns)] // FIXME: temporary
         if let Interpolated(nt) = &self.kind
-            && let NtExpr(_) | NtLiteral(_) | NtBlock(_) = &**nt
+            && let NtBlock(_) = &**nt
         {
             true
+        } else if matches!(
+            self.is_metavar_seq(),
+            Some(MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path)
+        ) {
+            true
         } else {
             matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
         }
     }
 
     /// Is the token an interpolated block (`$b:block`)?
     pub fn is_whole_block(&self) -> bool {
+        #[allow(irrefutable_let_patterns)] // FIXME: temporary
         if let Interpolated(nt) = &self.kind
             && let NtBlock(..) = &**nt
         {
@@ -1100,8 +1067,6 @@ pub enum NtExprKind {
 /// For interpolation during macro expansion.
 pub enum Nonterminal {
     NtBlock(P<ast::Block>),
-    NtExpr(P<ast::Expr>),
-    NtLiteral(P<ast::Expr>),
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -1191,15 +1156,12 @@ impl Nonterminal {
     pub fn use_span(&self) -> Span {
         match self {
             NtBlock(block) => block.span,
-            NtExpr(expr) | NtLiteral(expr) => expr.span,
         }
     }
 
     pub fn descr(&self) -> &'static str {
         match self {
             NtBlock(..) => "block",
-            NtExpr(..) => "expression",
-            NtLiteral(..) => "literal",
         }
     }
 }
@@ -1218,8 +1180,6 @@ impl fmt::Debug for Nonterminal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
             NtBlock(..) => f.pad("NtBlock(..)"),
-            NtExpr(..) => f.pad("NtExpr(..)"),
-            NtLiteral(..) => f.pad("NtLiteral(..)"),
         }
     }
 }
@@ -1242,7 +1202,7 @@ mod size_asserts {
     // tidy-alphabetical-start
     static_assert_size!(Lit, 12);
     static_assert_size!(LitKind, 2);
-    static_assert_size!(Nonterminal, 16);
+    static_assert_size!(Nonterminal, 8);
     static_assert_size!(Token, 24);
     static_assert_size!(TokenKind, 16);
     // tidy-alphabetical-end

compiler/rustc_ast/src/tokenstream.rs

@@ -233,35 +233,52 @@ fn attrs_and_tokens_to_token_trees(
 
     // Insert inner attribute tokens.
     if !inner_attrs.is_empty() {
-        let mut found = false;
-        // Check the last two trees (to account for a trailing semi)
-        for tree in res.iter_mut().rev().take(2) {
-            if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
-                // Inner attributes are only supported on extern blocks, functions,
-                // impls, and modules. All of these have their inner attributes
-                // placed at the beginning of the rightmost outermost braced group:
-                // e.g. fn foo() { #![my_attr] }
-                //
-                // Therefore, we can insert them back into the right location
-                // without needing to do any extra position tracking.
-                //
-                // Note: Outline modules are an exception - they can
-                // have attributes like `#![my_attr]` at the start of a file.
-                // Support for custom attributes in this position is not
-                // properly implemented - we always synthesize fake tokens,
-                // so we never reach this code.
+        let found = insert_inner_attrs(inner_attrs, res);
+        assert!(found, "Failed to find trailing delimited group in: {res:?}");
+    }
+
+    // Inner attributes are only supported on blocks, functions, impls, and
+    // modules. All of these have their inner attributes placed at the
+    // beginning of the rightmost outermost braced group:
+    // e.g. `fn foo() { #![my_attr] }`. (Note: the braces may be within
+    // invisible delimiters.)
+    //
+    // Therefore, we can insert them back into the right location without
+    // needing to do any extra position tracking.
+    //
+    // Note: Outline modules are an exception - they can have attributes like
+    // `#![my_attr]` at the start of a file. Support for custom attributes in
+    // this position is not properly implemented - we always synthesize fake
+    // tokens, so we never reach this code.
+    fn insert_inner_attrs(inner_attrs: &[Attribute], tts: &mut Vec<TokenTree>) -> bool {
+        for tree in tts.iter_mut().rev() {
+            if let TokenTree::Delimited(span, spacing, Delimiter::Brace, stream) = tree {
+                // Found it: the rightmost, outermost braced group.
                 let mut tts = vec![];
                 for inner_attr in inner_attrs {
                     tts.extend(inner_attr.token_trees());
                 }
-                tts.extend(delim_tokens.0.iter().cloned());
+                tts.extend(stream.0.iter().cloned());
                 let stream = TokenStream::new(tts);
-                *tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
-                found = true;
-                break;
+                *tree = TokenTree::Delimited(*span, *spacing, Delimiter::Brace, stream);
+                return true;
+            } else if let TokenTree::Delimited(span, spacing, Delimiter::Invisible(src), stream) =
+                tree
+            {
+                // Recurse inside invisible delimiters.
+                let mut vec: Vec<_> = stream.iter().cloned().collect();
+                if insert_inner_attrs(inner_attrs, &mut vec) {
+                    *tree = TokenTree::Delimited(
+                        *span,
+                        *spacing,
+                        Delimiter::Invisible(*src),
+                        TokenStream::new(vec),
+                    );
+                    return true;
+                }
             }
         }
-        assert!(found, "Failed to find trailing delimited group in: {res:?}");
+        false
     }
 }
 
@@ -462,7 +479,6 @@ impl TokenStream {
     pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
         match nt {
             Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
         }
     }
 

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -3,11 +3,10 @@ use std::sync::Arc;
 
 use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::token::{
-    self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
-    TokenKind,
+    self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Token, TokenKind,
 };
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
-use rustc_ast::{ExprKind, StmtKind, TyKind};
+use rustc_ast::{ExprKind, StmtKind, TyKind, UnOp};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
 use rustc_parse::lexer::nfc_normalize;
@@ -340,6 +339,30 @@ pub(super) fn transcribe<'a>(
                             MetaVarKind::Pat(*pat_kind),
                             TokenStream::from_ast(pat),
                         ),
+                        MatchedSingle(ParseNtResult::Expr(expr, kind)) => {
+                            let (can_begin_literal_maybe_minus, can_begin_string_literal) =
+                                match &expr.kind {
+                                    ExprKind::Lit(_) => (true, true),
+                                    ExprKind::Unary(UnOp::Neg, e)
+                                        if matches!(&e.kind, ExprKind::Lit(_)) =>
+                                    {
+                                        (true, false)
+                                    }
+                                    _ => (false, false),
+                                };
+                            mk_delimited(
+                                expr.span,
+                                MetaVarKind::Expr {
+                                    kind: *kind,
+                                    can_begin_literal_maybe_minus,
+                                    can_begin_string_literal,
+                                },
+                                TokenStream::from_ast(expr),
+                            )
+                        }
+                        MatchedSingle(ParseNtResult::Literal(lit)) => {
+                            mk_delimited(lit.span, MetaVarKind::Literal, TokenStream::from_ast(lit))
+                        }
                         MatchedSingle(ParseNtResult::Ty(ty)) => {
                             let is_path = matches!(&ty.kind, TyKind::Path(None, _path));
                             mk_delimited(
@@ -869,10 +892,8 @@ fn extract_symbol_from_pnr<'a>(
             },
             _,
         )) => Ok(*symbol),
-        ParseNtResult::Nt(nt)
-            if let Nonterminal::NtLiteral(expr) = &**nt
-                && let ExprKind::Lit(Lit { kind: LitKind::Str, symbol, suffix: None }) =
-                    &expr.kind =>
+        ParseNtResult::Literal(expr)
+            if let ExprKind::Lit(Lit { kind: LitKind::Str, symbol, suffix: None }) = &expr.kind =>
         {
             Ok(*symbol)
         }

compiler/rustc_parse/messages.ftl

@@ -858,7 +858,7 @@ parse_unexpected_parentheses_in_match_arm_pattern = unexpected parentheses surro
 parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
     .note = you cannot use `Self` as a generic parameter because it is reserved for associated items
 
-parse_unexpected_token_after_dot = unexpected token: `{$actual}`
+parse_unexpected_token_after_dot = unexpected token: {$actual}
 
 parse_unexpected_token_after_label = expected `while`, `for`, `loop` or `{"{"}` after a label
     .suggestion_remove_label = consider removing the label