Introduce InvisibleSource on invisible delimiters.

It's not used meaningfully yet, but will be needed to get rid of
interpolated tokens.

Author: nnethercote

compiler/rustc_ast/src/attr/mod.rs

@@ -371,7 +371,7 @@ impl MetaItemKind {
         tokens: &mut impl Iterator<Item = &'a TokenTree>,
     ) -> Option<MetaItemKind> {
         match tokens.next() {
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(_, Delimiter::Invisible(_), inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
             }
             Some(TokenTree::Token(token, _)) => {
@@ -510,7 +510,7 @@ impl NestedMetaItem {
                 tokens.next();
                 return Some(NestedMetaItem::Lit(lit));
             }
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(_, Delimiter::Invisible(_), inner_tokens)) => {
                 tokens.next();
                 return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
             }

compiler/rustc_ast/src/token.rs

@@ -55,7 +55,33 @@ pub enum Delimiter {
     /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
     /// `$var * 3` where `$var` is `1 + 2`.
     /// Invisible delimiters might not survive roundtrip of a token stream through a string.
-    Invisible,
+    Invisible(InvisibleSource),
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
+pub enum InvisibleSource {
+    // Converted from `proc_macro::Delimiter` in
+    // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
+    ProcMacro,
+
+    // Converted from `TokenKind::Interpolated` in
+    // `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
+    FlattenToken,
+}
+
+impl Delimiter {
+    // Should the parser skip these delimiters? Only happens for certain kinds
+    // of invisible delimiters. Once all interpolated nonterminals are removed,
+    // the answer should become `false` for all kinds, whereupon this function
+    // can be removed.
+    pub fn skip(&self) -> bool {
+        match self {
+            Delimiter::Invisible(src) => match src {
+                InvisibleSource::FlattenToken | InvisibleSource::ProcMacro => true,
+            },
+            Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
+        }
+    }
 }
 
 // Note that the suffix is *not* considered when deciding the `LitKind` in this

compiler/rustc_ast/src/tokenstream.rs

@@ -15,7 +15,7 @@
 
 use crate::ast::{AttrStyle, StmtKind};
 use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
-use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
+use crate::token::{self, Delimiter, InvisibleSource, Nonterminal, Token, TokenKind};
 use crate::AttrVec;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -486,7 +486,7 @@ impl TokenStream {
             }
             token::Interpolated(nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
-                Delimiter::Invisible,
+                Delimiter::Invisible(InvisibleSource::FlattenToken),
                 TokenStream::from_nonterminal_ast(nt).flattened(),
             ),
             _ => TokenTree::Token(token.clone(), spacing),

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -790,9 +790,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::CloseDelim(Delimiter::Bracket) => "]".into(),
             token::OpenDelim(Delimiter::Brace) => "{".into(),
             token::CloseDelim(Delimiter::Brace) => "}".into(),
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
-                "".into()
-            }
+            token::OpenDelim(Delimiter::Invisible(_))
+            | token::CloseDelim(Delimiter::Invisible(_)) => "".into(),
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -722,7 +722,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
                         let mbe::TokenTree::Token(bang) = bang &&
                         let TokenKind::Not = bang.kind &&
                         let mbe::TokenTree::Delimited(_, del) = args &&
-                        del.delim != Delimiter::Invisible
+                        !del.delim.skip()
                     {
                         true
                     } else {

compiler/rustc_expand/src/mbe/quoted.rs

@@ -151,11 +151,12 @@ fn parse_tree<'a>(
             // during parsing.
             let mut next = outer_trees.next();
             let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
-            if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
-                trees = Box::new(tts.trees());
-                next = trees.next();
-            } else {
-                trees = Box::new(outer_trees);
+            match next {
+                Some(tokenstream::TokenTree::Delimited(_, delim, tts)) if delim.skip() => {
+                    trees = Box::new(tts.trees());
+                    next = trees.next();
+                }
+                _ => trees = Box::new(outer_trees),
             }
 
             match next {

compiler/rustc_expand/src/proc_macro_server.rs

@@ -34,7 +34,7 @@ impl FromInternal<token::Delimiter> for Delimiter {
             token::Delimiter::Parenthesis => Delimiter::Parenthesis,
             token::Delimiter::Brace => Delimiter::Brace,
             token::Delimiter::Bracket => Delimiter::Bracket,
-            token::Delimiter::Invisible => Delimiter::None,
+            token::Delimiter::Invisible(_) => Delimiter::None,
         }
     }
 }
@@ -45,7 +45,7 @@ impl ToInternal<token::Delimiter> for Delimiter {
             Delimiter::Parenthesis => token::Delimiter::Parenthesis,
             Delimiter::Brace => token::Delimiter::Brace,
             Delimiter::Bracket => token::Delimiter::Bracket,
-            Delimiter::None => token::Delimiter::Invisible,
+            Delimiter::None => token::Delimiter::Invisible(token::InvisibleSource::ProcMacro),
         }
     }
 }

compiler/rustc_parse/src/parser/mod.rs

@@ -263,7 +263,7 @@ impl TokenCursor {
                     &TokenTree::Delimited(sp, delim, ref tts) => {
                         let trees = tts.clone().into_trees();
                         self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp));
-                        if delim != Delimiter::Invisible {
+                        if !delim.skip() {
                             return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
                         }
                         // No open delimiter to return; continue on to the next iteration.
@@ -272,7 +272,7 @@ impl TokenCursor {
             } else if let Some((tree_cursor, delim, span)) = self.stack.pop() {
                 // We have exhausted this token stream. Move back to its parent token stream.
                 self.tree_cursor = tree_cursor;
-                if delim != Delimiter::Invisible {
+                if !delim.skip() {
                     return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
                 }
                 // No close delimiter to return; continue on to the next iteration.
@@ -1046,7 +1046,7 @@ impl<'a> Parser<'a> {
         }
         debug_assert!(!matches!(
             next.0.kind,
-            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+            token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
         ));
         self.inlined_bump_with(next)
     }
@@ -1060,17 +1060,17 @@ impl<'a> Parser<'a> {
         }
 
         if let Some(&(_, delim, span)) = self.token_cursor.stack.last()
-            && delim != Delimiter::Invisible
+            && !delim.skip()
         {
             // We are not in the outermost token stream, and the token stream
             // we are in has non-skipped delimiters. Look for skipped
             // delimiters in the lookahead range.
             let tree_cursor = &self.token_cursor.tree_cursor;
-            let all_normal = (0..dist).all(|i| {
+            let any_skip = (0..dist).any(|i| {
                 let token = tree_cursor.look_ahead(i);
-                !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
+                matches!(token, Some(TokenTree::Delimited(_, delim, _)) if delim.skip())
             });
-            if all_normal {
+            if !any_skip {
                 // There were no skipped delimiters. Do lookahead by plain indexing.
                 return match tree_cursor.look_ahead(dist - 1) {
                     Some(tree) => {
@@ -1101,7 +1101,7 @@ impl<'a> Parser<'a> {
             token = cursor.next().0;
             if matches!(
                 token.kind,
-                token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
+                token::OpenDelim(delim) | token::CloseDelim(delim) if delim.skip()
             ) {
                 continue;
             }

src/tools/rustfmt/src/macros.rs

@@ -563,7 +563,7 @@ fn delim_token_to_str(
                 ("{ ", " }")
             }
         }
-        Delimiter::Invisible => unreachable!(),
+        Delimiter::Invisible(_) => unreachable!(),
     };
     if use_multiple_lines {
         let indent_str = shape.indent.to_string_with_newline(context.config);