Introduce InvisibleOrigin on invisible delimiters.

nnethercote · nnethercote · commit 74d42f4152b1 · 2024-05-16T11:05:49.000+10:00
It's not used meaningfully yet, but will be needed to get rid of
interpolated tokens.
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
@@ -383,7 +383,7 @@ impl MetaItemKind {
         tokens: &mut impl Iterator<Item = &'a TokenTree>,
     ) -> Option<MetaItemKind> {
         match tokens.next() {
-            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
             }
             Some(TokenTree::Token(token, _)) => {
@@ -521,7 +521,7 @@ impl NestedMetaItem {
                 tokens.next();
                 return Some(NestedMetaItem::Lit(lit));
             }
-            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
                 tokens.next();
                 return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
             }
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
@@ -39,6 +39,20 @@ pub enum BinOpToken {
     Shr,
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
+pub enum InvisibleOrigin {
+    // From the expansion of a metavariable in a declarative macro.
+    MetaVar(NonterminalKind),
+
+    // Converted from `proc_macro::Delimiter` in
+    // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
+    ProcMacro,
+
+    // Converted from `TokenKind::Interpolated` in
+    // `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
+    FlattenToken,
+}
+
 /// Describes how a sequence of token trees is delimited.
 /// Cannot use `proc_macro::Delimiter` directly because this
 /// structure should implement some additional traits.
@@ -56,7 +70,22 @@ pub enum Delimiter {
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
     /// Invisible delimiters might not survive roundtrip of a token stream through a string.
-    Invisible,
+    Invisible(InvisibleOrigin),
+}
+
+impl Delimiter {
+    // Should the parser skip these delimiters? Only happens for certain kinds
+    // of invisible delimiters. Ideally this function will eventually disappear
+    // and no invisible delimiters will be skipped.
+    #[inline]
+    pub fn skip(&self) -> bool {
+        use InvisibleOrigin::*;
+        match self {
+            Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
+            Delimiter::Invisible(MetaVar(_)) => false,
+            Delimiter::Invisible(FlattenToken | ProcMacro) => true,
+        }
+    }
 }
 
 // Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -869,7 +898,7 @@ pub enum Nonterminal {
     NtVis(P<ast::Visibility>),
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NonterminalKind {
     Item,
     Block,
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
@@ -15,7 +15,7 @@
 
 use crate::ast::{AttrStyle, StmtKind};
 use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
-use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
+use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
 use crate::AttrVec;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -490,13 +490,13 @@ impl TokenStream {
             token::NtLifetime(ident) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
-                Delimiter::Invisible,
+                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
                 TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
             ),
             token::Interpolated(ref nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
-                Delimiter::Invisible,
+                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
                 TokenStream::from_nonterminal_ast(&nt).flattened(),
             ),
             _ => TokenTree::Token(token.clone(), spacing),
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -524,7 +524,10 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         }
     }
 
-    fn peek_comment<'b>(&'b self) -> Option<&'b Comment> where 'a: 'b {
+    fn peek_comment<'b>(&'b self) -> Option<&'b Comment>
+    where
+        'a: 'b,
+    {
         self.comments().and_then(|c| c.peek())
     }
 
@@ -907,9 +910,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::CloseDelim(Delimiter::Bracket) => "]".into(),
             token::OpenDelim(Delimiter::Brace) => "{".into(),
             token::CloseDelim(Delimiter::Brace) => "}".into(),
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
-                "".into()
-            }
+            token::OpenDelim(Delimiter::Invisible(_))
+            | token::CloseDelim(Delimiter::Invisible(_)) => "".into(),
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -702,7 +702,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
                     && let mbe::TokenTree::Token(bang) = bang
                     && let TokenKind::Not = bang.kind
                     && let mbe::TokenTree::Delimited(.., del) = args
-                    && del.delim != Delimiter::Invisible
+                    && !del.delim.skip()
                 {
                     true
                 } else {
diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs
@@ -152,11 +152,12 @@ fn parse_tree<'a>(
             // during parsing.
             let mut next = outer_trees.next();
             let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
-            if let Some(tokenstream::TokenTree::Delimited(.., Delimiter::Invisible, tts)) = next {
-                trees = Box::new(tts.trees());
-                next = trees.next();
-            } else {
-                trees = Box::new(outer_trees);
+            match next {
+                Some(tokenstream::TokenTree::Delimited(.., delim, tts)) if delim.skip() => {
+                    trees = Box::new(tts.trees());
+                    next = trees.next();
+                }
+                _ => trees = Box::new(outer_trees),
             }
 
             match next {
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -35,7 +35,7 @@ impl FromInternal<token::Delimiter> for Delimiter {
             token::Delimiter::Parenthesis => Delimiter::Parenthesis,
             token::Delimiter::Brace => Delimiter::Brace,
             token::Delimiter::Bracket => Delimiter::Bracket,
-            token::Delimiter::Invisible => Delimiter::None,
+            token::Delimiter::Invisible(_) => Delimiter::None,
         }
     }
 }
@@ -46,7 +46,7 @@ impl ToInternal<token::Delimiter> for Delimiter {
             Delimiter::Parenthesis => token::Delimiter::Parenthesis,
             Delimiter::Brace => token::Delimiter::Brace,
             Delimiter::Bracket => token::Delimiter::Bracket,
-            Delimiter::None => token::Delimiter::Invisible,
+            Delimiter::None => token::Delimiter::Invisible(token::InvisibleOrigin::ProcMacro),
         }
     }
 }
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
@@ -286,7 +286,7 @@ impl TokenCursor {
                             spacing,
                             delim,
                         ));
-                        if delim != Delimiter::Invisible {
+                        if !delim.skip() {
                             return (Token::new(token::OpenDelim(delim), sp.open), spacing.open);
                         }
                         // No open delimiter to return; continue on to the next iteration.
@@ -295,7 +295,7 @@ impl TokenCursor {
             } else if let Some((tree_cursor, span, spacing, delim)) = self.stack.pop() {
                 // We have exhausted this token stream. Move back to its parent token stream.
                 self.tree_cursor = tree_cursor;
-                if delim != Delimiter::Invisible {
+                if !delim.skip() {
                     return (Token::new(token::CloseDelim(delim), span.close), spacing.close);
                 }
                 // No close delimiter to return; continue on to the next iteration.
@@ -1113,7 +1113,7 @@ impl<'a> Parser<'a> {
         }
         debug_assert!(!matches!(
             next.0.kind,
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+            token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
         ));
         self.inlined_bump_with(next)
     }
@@ -1127,17 +1127,17 @@ impl<'a> Parser<'a> {
         }
 
         if let Some(&(_, span, _, delim)) = self.token_cursor.stack.last()
-            && delim != Delimiter::Invisible
+            && !delim.skip()
         {
             // We are not in the outermost token stream, and the token stream
             // we are in has non-skipped delimiters. Look for skipped
             // delimiters in the lookahead range.
             let tree_cursor = &self.token_cursor.tree_cursor;
-            let all_normal = (0..dist).all(|i| {
+            let any_skip = (0..dist).any(|i| {
                 let token = tree_cursor.look_ahead(i);
-                !matches!(token, Some(TokenTree::Delimited(.., Delimiter::Invisible, _)))
+                matches!(token, Some(TokenTree::Delimited(.., delim, _)) if delim.skip())
             });
-            if all_normal {
+            if !any_skip {
                 // There were no skipped delimiters. Do lookahead by plain indexing.
                 return match tree_cursor.look_ahead(dist - 1) {
                     Some(tree) => {
@@ -1168,7 +1168,7 @@ impl<'a> Parser<'a> {
             token = cursor.next().0;
             if matches!(
                 token.kind,
-                token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+                token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
             ) {
                 continue;
             }
diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs
@@ -587,7 +587,7 @@ fn delim_token_to_str(
                 ("{ ", " }")
             }
         }
-        Delimiter::Invisible => unreachable!(),
+        Delimiter::Invisible(_) => unreachable!(),
     };
     if use_multiple_lines {
         let indent_str = shape.indent.to_string_with_newline(context.config);

Original file line number	Diff line number	Diff line change
`@@ -383,7 +383,7 @@ impl MetaItemKind {`
`383`	`383`	`tokens: &mut impl Iterator<Item = &'a TokenTree>,`
`384`	`384`	`) -> Option<MetaItemKind> {`
`385`	`385`	`match tokens.next() {`
`386`		`- Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {`
	`386`	`+ Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {`
`387`	`387`	`MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())`
`388`	`388`	`}`
`389`	`389`	`Some(TokenTree::Token(token, _)) => {`
`@@ -521,7 +521,7 @@ impl NestedMetaItem {`
`521`	`521`	`tokens.next();`
`522`	`522`	`return Some(NestedMetaItem::Lit(lit));`
`523`	`523`	`}`
`524`		`- Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {`
	`524`	`+ Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {`
`525`	`525`	`tokens.next();`
`526`	`526`	`return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());`
`527`	`527`	`}`
Original file line number	Diff line number	Diff line change
`@@ -702,7 +702,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {`
`702`	`702`	`&& let mbe::TokenTree::Token(bang) = bang`
`703`	`703`	`&& let TokenKind::Not = bang.kind`
`704`	`704`	`&& let mbe::TokenTree::Delimited(.., del) = args`
`705`		`- && del.delim != Delimiter::Invisible`
	`705`	`+ && !del.delim.skip()`
`706`	`706`	`{`
`707`	`707`	`true`
`708`	`708`	`} else {`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ impl FromInternal<token::Delimiter> for Delimiter {`
`35`	`35`	`token::Delimiter::Parenthesis => Delimiter::Parenthesis,`
`36`	`36`	`token::Delimiter::Brace => Delimiter::Brace,`
`37`	`37`	`token::Delimiter::Bracket => Delimiter::Bracket,`
`38`		`- token::Delimiter::Invisible => Delimiter::None,`
	`38`	`+ token::Delimiter::Invisible(_) => Delimiter::None,`
`39`	`39`	`}`
`40`	`40`	`}`
`41`	`41`	`}`
`@@ -46,7 +46,7 @@ impl ToInternal<token::Delimiter> for Delimiter {`
`46`	`46`	`Delimiter::Parenthesis => token::Delimiter::Parenthesis,`
`47`	`47`	`Delimiter::Brace => token::Delimiter::Brace,`
`48`	`48`	`Delimiter::Bracket => token::Delimiter::Bracket,`
`49`		`- Delimiter::None => token::Delimiter::Invisible,`
	`49`	`+ Delimiter::None => token::Delimiter::Invisible(token::InvisibleOrigin::ProcMacro),`
`50`	`50`	`}`
`51`	`51`	`}`
`52`	`52`	`}`
Original file line number	Diff line number	Diff line change
`@@ -587,7 +587,7 @@ fn delim_token_to_str(`
`587`	`587`	`("{ ", " }")`
`588`	`588`	`}`
`589`	`589`	`}`
`590`		`- Delimiter::Invisible => unreachable!(),`
	`590`	`+ Delimiter::Invisible(_) => unreachable!(),`
`591`	`591`	`};`
`592`	`592`	`if use_multiple_lines {`
`593`	`593`	`let indent_str = shape.indent.to_string_with_newline(context.config);`