Add metavariables to TokenDescription.

Pasted metavariables are wrapped in invisible delimiters, which
pretty-print as empty strings, and changing that can break some proc
macros. But error messages saying "expected identifer, found ``" are
bad. So this commit adds support for metavariables in `TokenDescription`
so they print as "metavariable" in error messages, instead of "``".

It's not used meaningfully yet, but will be needed to get rid of
interpolated tokens.

Author: nnethercote

compiler/rustc_parse/messages.ftl

@@ -216,6 +216,9 @@ parse_expected_identifier_found_doc_comment = expected identifier, found doc com
 parse_expected_identifier_found_doc_comment_str = expected identifier, found doc comment `{$token}`
 parse_expected_identifier_found_keyword = expected identifier, found keyword
 parse_expected_identifier_found_keyword_str = expected identifier, found keyword `{$token}`
+parse_expected_identifier_found_metavar = expected identifier, found metavariable
+# This one deliberately doesn't print a token.
+parse_expected_identifier_found_metavar_str = expected identifier, found metavariable
 parse_expected_identifier_found_reserved_identifier = expected identifier, found reserved identifier
 parse_expected_identifier_found_reserved_identifier_str = expected identifier, found reserved identifier `{$token}`
 parse_expected_identifier_found_reserved_keyword = expected identifier, found reserved keyword
@@ -227,6 +230,8 @@ parse_expected_mut_or_const_in_raw_pointer_type = expected `mut` or `const` keyw
 
 parse_expected_semi_found_doc_comment_str = expected `;`, found doc comment `{$token}`
 parse_expected_semi_found_keyword_str = expected `;`, found keyword `{$token}`
+# This one deliberately doesn't print a token.
+parse_expected_semi_found_metavar_str = expected `;`, found metavariable
 parse_expected_semi_found_reserved_identifier_str = expected `;`, found reserved identifier `{$token}`
 parse_expected_semi_found_reserved_keyword_str = expected `;`, found reserved keyword `{$token}`
 parse_expected_semi_found_str = expected `;`, found `{$token}`
@@ -864,6 +869,8 @@ parse_unexpected_token_after_not_logical = use `!` to perform logical negation
 parse_unexpected_token_after_struct_name = expected `where`, `{"{"}`, `(`, or `;` after struct name
 parse_unexpected_token_after_struct_name_found_doc_comment = expected `where`, `{"{"}`, `(`, or `;` after struct name, found doc comment `{$token}`
 parse_unexpected_token_after_struct_name_found_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found keyword `{$token}`
+# This one deliberately doesn't print a token.
+parse_unexpected_token_after_struct_name_found_metavar = expected `where`, `{"{"}`, `(`, or `;` after struct name, found metavar
 parse_unexpected_token_after_struct_name_found_other = expected `where`, `{"{"}`, `(`, or `;` after struct name, found `{$token}`
 
 parse_unexpected_token_after_struct_name_found_reserved_identifier = expected `where`, `{"{"}`, `(`, or `;` after struct name, found reserved identifier `{$token}`

compiler/rustc_parse/src/errors.rs

@@ -1086,6 +1086,8 @@ pub(crate) enum ExpectedIdentifierFound {
     ReservedKeyword(#[primary_span] Span),
     #[label(parse_expected_identifier_found_doc_comment)]
     DocComment(#[primary_span] Span),
+    #[label(parse_expected_identifier_found_metavar)]
+    MetaVar(#[primary_span] Span),
     #[label(parse_expected_identifier)]
     Other(#[primary_span] Span),
 }
@@ -1099,6 +1101,7 @@ impl ExpectedIdentifierFound {
             Some(TokenDescription::Keyword) => ExpectedIdentifierFound::Keyword,
             Some(TokenDescription::ReservedKeyword) => ExpectedIdentifierFound::ReservedKeyword,
             Some(TokenDescription::DocComment) => ExpectedIdentifierFound::DocComment,
+            Some(TokenDescription::MetaVar(_)) => ExpectedIdentifierFound::MetaVar,
             None => ExpectedIdentifierFound::Other,
         })(span)
     }
@@ -1117,6 +1120,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedIdentifier {
     fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> {
         let token_descr = TokenDescription::from_token(&self.token);
 
+        let mut add_token = true;
         let mut diag = Diag::new(dcx, level, match token_descr {
             Some(TokenDescription::ReservedIdentifier) => {
                 fluent::parse_expected_identifier_found_reserved_identifier_str
@@ -1128,10 +1132,16 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedIdentifier {
             Some(TokenDescription::DocComment) => {
                 fluent::parse_expected_identifier_found_doc_comment_str
             }
+            Some(TokenDescription::MetaVar(_)) => {
+                add_token = false;
+                fluent::parse_expected_identifier_found_metavar_str
+            }
             None => fluent::parse_expected_identifier_found_str,
         });
         diag.span(self.span);
-        diag.arg("token", self.token);
+        if add_token {
+            diag.arg("token", self.token);
+        }
 
         if let Some(sugg) = self.suggest_raw {
             sugg.add_to_diag(&mut diag);
@@ -1171,6 +1181,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedSemi {
     fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> {
         let token_descr = TokenDescription::from_token(&self.token);
 
+        let mut add_token = true;
         let mut diag = Diag::new(dcx, level, match token_descr {
             Some(TokenDescription::ReservedIdentifier) => {
                 fluent::parse_expected_semi_found_reserved_identifier_str
@@ -1180,10 +1191,16 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for ExpectedSemi {
                 fluent::parse_expected_semi_found_reserved_keyword_str
             }
             Some(TokenDescription::DocComment) => fluent::parse_expected_semi_found_doc_comment_str,
+            Some(TokenDescription::MetaVar(_)) => {
+                add_token = false;
+                fluent::parse_expected_semi_found_metavar_str
+            }
             None => fluent::parse_expected_semi_found_str,
         });
         diag.span(self.span);
-        diag.arg("token", self.token);
+        if add_token {
+            diag.arg("token", self.token);
+        }
 
         if let Some(unexpected_token_label) = self.unexpected_token_label {
             diag.span_label(unexpected_token_label, fluent::parse_label_unexpected_token);
@@ -1925,6 +1942,12 @@ pub(crate) enum UnexpectedTokenAfterStructName {
         span: Span,
         token: Token,
     },
+    #[diag(parse_unexpected_token_after_struct_name_found_metavar)]
+    MetaVar {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+    },
     #[diag(parse_unexpected_token_after_struct_name_found_other)]
     Other {
         #[primary_span]
@@ -1941,6 +1964,7 @@ impl UnexpectedTokenAfterStructName {
             Some(TokenDescription::Keyword) => Self::Keyword { span, token },
             Some(TokenDescription::ReservedKeyword) => Self::ReservedKeyword { span, token },
             Some(TokenDescription::DocComment) => Self::DocComment { span, token },
+            Some(TokenDescription::MetaVar(_)) => Self::MetaVar { span },
             None => Self::Other { span, token },
         }
     }

compiler/rustc_parse/src/parser/mod.rs

@@ -21,7 +21,9 @@ pub(crate) use item::FnParseMode;
 pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
 use path::PathStyle;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
+use rustc_ast::token::{
+    self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, Token, TokenKind,
+};
 use rustc_ast::tokenstream::{
     AttrsTarget, DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree, TokenTreeCursor,
 };
@@ -410,6 +412,12 @@ pub(super) enum TokenDescription {
     Keyword,
     ReservedKeyword,
     DocComment,
+
+    // Expanded metavariables are wrapped in invisible delimiters which aren't
+    // pretty-printed. In error messages we must handle these specially
+    // otherwise we get confusing things in messages like "expected `(`, found
+    // ``". It's better to say e.g. "expected `(`, found type metavariable".
+    MetaVar(MetaVarKind),
 }
 
 impl TokenDescription {
@@ -419,26 +427,29 @@ impl TokenDescription {
             _ if token.is_used_keyword() => Some(TokenDescription::Keyword),
             _ if token.is_unused_keyword() => Some(TokenDescription::ReservedKeyword),
             token::DocComment(..) => Some(TokenDescription::DocComment),
+            token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
+                Some(TokenDescription::MetaVar(kind))
+            }
             _ => None,
         }
     }
 }
 
 pub fn token_descr(token: &Token) -> String {
-    let name = pprust::token_to_string(token).to_string();
-
-    let kind = match (TokenDescription::from_token(token), &token.kind) {
-        (Some(TokenDescription::ReservedIdentifier), _) => Some("reserved identifier"),
-        (Some(TokenDescription::Keyword), _) => Some("keyword"),
-        (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
-        (Some(TokenDescription::DocComment), _) => Some("doc comment"),
-        (None, TokenKind::NtIdent(..)) => Some("identifier"),
-        (None, TokenKind::NtLifetime(..)) => Some("lifetime"),
-        (None, TokenKind::Interpolated(node)) => Some(node.descr()),
-        (None, _) => None,
-    };
-
-    if let Some(kind) = kind { format!("{kind} `{name}`") } else { format!("`{name}`") }
+    let s = pprust::token_to_string(token).to_string();
+
+    match (TokenDescription::from_token(token), &token.kind) {
+        (Some(TokenDescription::ReservedIdentifier), _) => format!("reserved identifier `{s}`"),
+        (Some(TokenDescription::Keyword), _) => format!("keyword `{s}`"),
+        (Some(TokenDescription::ReservedKeyword), _) => format!("reserved keyword `{s}`"),
+        (Some(TokenDescription::DocComment), _) => format!("doc comment `{s}`"),
+        // Deliberately doesn't print `s`, which is empty.
+        (Some(TokenDescription::MetaVar(kind)), _) => format!("`{kind}` metavariable"),
+        (None, TokenKind::NtIdent(..)) => format!("identifier `{s}`"),
+        (None, TokenKind::NtLifetime(..)) => format!("lifetime `{s}`"),
+        (None, TokenKind::Interpolated(node)) => format!("{} `{s}`", node.descr()),
+        (None, _) => format!("`{s}`"),
+    }
 }
 
 impl<'a> Parser<'a> {