Introduce InvisibleOrigin on invisible delimiters.

It's not used meaningfully yet, but will be needed to get rid of
interpolated tokens.

Author: nnethercote

compiler/rustc_ast/src/attr/mod.rs

@@ -404,7 +404,7 @@ impl MetaItemKind {
         tokens: &mut impl Iterator<Item = &'a TokenTree>,
     ) -> Option<MetaItemKind> {
         match tokens.next() {
-            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
             }
             Some(TokenTree::Token(token, _)) => {
@@ -542,7 +542,7 @@ impl NestedMetaItem {
                 tokens.next();
                 return Some(NestedMetaItem::Lit(lit));
             }
-            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
                 tokens.next();
                 return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
             }

compiler/rustc_ast/src/token.rs

@@ -41,6 +41,20 @@ pub enum BinOpToken {
     Shr,
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
+pub enum InvisibleOrigin {
+    // From the expansion of a metavariable in a declarative macro.
+    MetaVar(NonterminalKind),
+
+    // Converted from `proc_macro::Delimiter` in
+    // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
+    ProcMacro,
+
+    // Converted from `TokenKind::Interpolated` in
+    // `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
+    FlattenToken,
+}
+
 /// Describes how a sequence of token trees is delimited.
 /// Cannot use `proc_macro::Delimiter` directly because this
 /// structure should implement some additional traits.
@@ -58,7 +72,22 @@ pub enum Delimiter {
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
     /// Invisible delimiters might not survive roundtrip of a token stream through a string.
-    Invisible,
+    Invisible(InvisibleOrigin),
+}
+
+impl Delimiter {
+    // Should the parser skip these delimiters? Only happens for certain kinds
+    // of invisible delimiters. Ideally this function will eventually disappear
+    // and no invisible delimiters will be skipped.
+    #[inline]
+    pub fn skip(&self) -> bool {
+        use InvisibleOrigin::*;
+        match self {
+            Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
+            Delimiter::Invisible(MetaVar(_)) => false,
+            Delimiter::Invisible(FlattenToken | ProcMacro) => true,
+        }
+    }
 }
 
 // Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -873,7 +902,7 @@ impl PartialEq<TokenKind> for Token {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NtPatKind {
     // Matches or-patterns. Was written using `pat` in edition 2021 or later.
     PatWithOr,
@@ -883,7 +912,7 @@ pub enum NtPatKind {
     PatParam { inferred: bool },
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NtExprKind {
     // Matches expressions using the post-edition 2024. Was written using
     // `expr` in edition 2024 or later.
@@ -910,7 +939,7 @@ pub enum Nonterminal {
     NtVis(P<ast::Visibility>),
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NonterminalKind {
     Item,
     Block,

compiler/rustc_ast/src/tokenstream.rs

@@ -15,7 +15,7 @@
 
 use crate::ast::{AttrStyle, StmtKind};
 use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
-use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
+use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
 use crate::AttrVec;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -490,13 +490,13 @@ impl TokenStream {
             token::NtLifetime(ident) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
-                Delimiter::Invisible,
+                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
                 TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
             ),
             token::Interpolated(ref nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
-                Delimiter::Invisible,
+                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
                 TokenStream::from_nonterminal_ast(&nt).flattened(),
             ),
             _ => TokenTree::Token(token.clone(), spacing),

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -928,9 +928,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::CloseDelim(Delimiter::Bracket) => "]".into(),
             token::OpenDelim(Delimiter::Brace) => "{".into(),
             token::CloseDelim(Delimiter::Brace) => "}".into(),
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
-                "".into()
-            }
+            token::OpenDelim(Delimiter::Invisible(_))
+            | token::CloseDelim(Delimiter::Invisible(_)) => "".into(),
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -707,7 +707,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
                     && let mbe::TokenTree::Token(bang) = bang
                     && let TokenKind::Not = bang.kind
                     && let mbe::TokenTree::Delimited(.., del) = args
-                    && del.delim != Delimiter::Invisible
+                    && !del.delim.skip()
                 {
                     true
                 } else {

compiler/rustc_expand/src/mbe/quoted.rs

@@ -189,11 +189,12 @@ fn parse_tree<'a>(
             // during parsing.
             let mut next = outer_trees.next();
             let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
-            if let Some(tokenstream::TokenTree::Delimited(.., Delimiter::Invisible, tts)) = next {
-                trees = Box::new(tts.trees());
-                next = trees.next();
-            } else {
-                trees = Box::new(outer_trees);
+            match next {
+                Some(tokenstream::TokenTree::Delimited(.., delim, tts)) if delim.skip() => {
+                    trees = Box::new(tts.trees());
+                    next = trees.next();
+                }
+                _ => trees = Box::new(outer_trees),
             }
 
             match next {

compiler/rustc_expand/src/proc_macro_server.rs

@@ -36,7 +36,7 @@ impl FromInternal<token::Delimiter> for Delimiter {
             token::Delimiter::Parenthesis => Delimiter::Parenthesis,
             token::Delimiter::Brace => Delimiter::Brace,
             token::Delimiter::Bracket => Delimiter::Bracket,
-            token::Delimiter::Invisible => Delimiter::None,
+            token::Delimiter::Invisible(_) => Delimiter::None,
         }
     }
 }
@@ -47,7 +47,7 @@ impl ToInternal<token::Delimiter> for Delimiter {
             Delimiter::Parenthesis => token::Delimiter::Parenthesis,
             Delimiter::Brace => token::Delimiter::Brace,
             Delimiter::Bracket => token::Delimiter::Bracket,
-            Delimiter::None => token::Delimiter::Invisible,
+            Delimiter::None => token::Delimiter::Invisible(token::InvisibleOrigin::ProcMacro),
         }
     }
 }

compiler/rustc_parse/src/parser/mod.rs

@@ -285,7 +285,7 @@ impl TokenCursor {
                             spacing,
                             delim,
                         ));
-                        if delim != Delimiter::Invisible {
+                        if !delim.skip() {
                             return (Token::new(token::OpenDelim(delim), sp.open), spacing.open);
                         }
                         // No open delimiter to return; continue on to the next iteration.
@@ -294,7 +294,7 @@ impl TokenCursor {
             } else if let Some((tree_cursor, span, spacing, delim)) = self.stack.pop() {
                 // We have exhausted this token stream. Move back to its parent token stream.
                 self.tree_cursor = tree_cursor;
-                if delim != Delimiter::Invisible {
+                if !delim.skip() {
                     return (Token::new(token::CloseDelim(delim), span.close), spacing.close);
                 }
                 // No close delimiter to return; continue on to the next iteration.
@@ -1105,7 +1105,7 @@ impl<'a> Parser<'a> {
         }
         debug_assert!(!matches!(
             next.0.kind,
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+            token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
         ));
         self.inlined_bump_with(next)
     }
@@ -1119,17 +1119,17 @@ impl<'a> Parser<'a> {
         }
 
         if let Some(&(_, span, _, delim)) = self.token_cursor.stack.last()
-            && delim != Delimiter::Invisible
+            && !delim.skip()
         {
             // We are not in the outermost token stream, and the token stream
             // we are in has non-skipped delimiters. Look for skipped
             // delimiters in the lookahead range.
             let tree_cursor = &self.token_cursor.tree_cursor;
-            let all_normal = (0..dist).all(|i| {
+            let any_skip = (0..dist).any(|i| {
                 let token = tree_cursor.look_ahead(i);
-                !matches!(token, Some(TokenTree::Delimited(.., Delimiter::Invisible, _)))
+                matches!(token, Some(TokenTree::Delimited(.., delim, _)) if delim.skip())
             });
-            if all_normal {
+            if !any_skip {
                 // There were no skipped delimiters. Do lookahead by plain indexing.
                 return match tree_cursor.look_ahead(dist - 1) {
                     Some(tree) => {
@@ -1160,7 +1160,7 @@ impl<'a> Parser<'a> {
             token = cursor.next().0;
             if matches!(
                 token.kind,
-                token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+                token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
             ) {
                 continue;
             }

src/tools/rustfmt/src/macros.rs

@@ -588,7 +588,7 @@ fn delim_token_to_str(
                 ("{ ", " }")
             }
         }
-        Delimiter::Invisible => unreachable!(),
+        Delimiter::Invisible(_) => unreachable!(),
     };
     if use_multiple_lines {
         let indent_str = shape.indent.to_string_with_newline(context.config);