Auto merge of #103898 - Nilstrieb:match-macro, r=nnethercote

Retry failed macro matching for diagnostics

When a declarative macro fails to match, retry the matching to collect diagnostic info instead of collecting it on the fly in the hot path. Split out of #103439.

You made a bunch of changes to declarative macro matching, so
r? `@nnethercote`

This change should produce a few small perf wins: #103439 (comment)

Author: bors

compiler/rustc_expand/src/mbe.rs

@@ -52,7 +52,7 @@ impl KleeneToken {
 /// A Kleene-style [repetition operator](https://en.wikipedia.org/wiki/Kleene_star)
 /// for token sequences.
 #[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy)]
-enum KleeneOp {
+pub(crate) enum KleeneOp {
     /// Kleene star (`*`) for zero or more repetitions
     ZeroOrMore,
     /// Kleene plus (`+`) for one or more repetitions

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -73,17 +73,17 @@
 pub(crate) use NamedMatch::*;
 pub(crate) use ParseResult::*;
 
-use crate::mbe::{KleeneOp, TokenTree};
+use crate::mbe::{macro_rules::Tracker, KleeneOp, TokenTree};
 
 use rustc_ast::token::{self, DocComment, Nonterminal, NonterminalKind, Token};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Lrc;
+use rustc_errors::ErrorGuaranteed;
 use rustc_lint_defs::pluralize;
 use rustc_parse::parser::{NtOrTt, Parser};
+use rustc_span::symbol::Ident;
 use rustc_span::symbol::MacroRulesNormalizedIdent;
 use rustc_span::Span;
-
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::sync::Lrc;
-use rustc_span::symbol::Ident;
 use std::borrow::Cow;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
 
@@ -96,7 +96,8 @@ use std::collections::hash_map::Entry::{Occupied, Vacant};
 ///
 /// This means a matcher can be represented by `&[MatcherLoc]`, and traversal mostly involves
 /// simply incrementing the current matcher position index by one.
-pub(super) enum MatcherLoc {
+#[derive(Debug)]
+pub(crate) enum MatcherLoc {
     Token {
         token: Token,
     },
@@ -270,13 +271,17 @@ pub(crate) enum ParseResult<T> {
     Failure(Token, &'static str),
     /// Fatal error (malformed macro?). Abort compilation.
     Error(rustc_span::Span, String),
-    ErrorReported,
+    ErrorReported(ErrorGuaranteed),
 }
 
 /// A `ParseResult` where the `Success` variant contains a mapping of
 /// `MacroRulesNormalizedIdent`s to `NamedMatch`es. This represents the mapping
 /// of metavars to the token trees they bind to.
-pub(crate) type NamedParseResult = ParseResult<FxHashMap<MacroRulesNormalizedIdent, NamedMatch>>;
+pub(crate) type NamedParseResult = ParseResult<NamedMatches>;
+
+/// Contains a mapping of `MacroRulesNormalizedIdent`s to `NamedMatch`es.
+/// This represents the mapping of metavars to the token trees they bind to.
+pub(crate) type NamedMatches = FxHashMap<MacroRulesNormalizedIdent, NamedMatch>;
 
 /// Count how many metavars declarations are in `matcher`.
 pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize {
@@ -400,17 +405,21 @@ impl TtParser {
     ///
     /// `Some(result)` if everything is finished, `None` otherwise. Note that matches are kept
     /// track of through the mps generated.
-    fn parse_tt_inner(
+    fn parse_tt_inner<'matcher, T: Tracker<'matcher>>(
         &mut self,
-        matcher: &[MatcherLoc],
+        matcher: &'matcher [MatcherLoc],
         token: &Token,
+        track: &mut T,
     ) -> Option<NamedParseResult> {
         // Matcher positions that would be valid if the macro invocation was over now. Only
         // modified if `token == Eof`.
         let mut eof_mps = EofMatcherPositions::None;
 
         while let Some(mut mp) = self.cur_mps.pop() {
-            match &matcher[mp.idx] {
+            let matcher_loc = &matcher[mp.idx];
+            track.before_match_loc(self, matcher_loc);
+
+            match matcher_loc {
                 MatcherLoc::Token { token: t } => {
                     // If it's a doc comment, we just ignore it and move on to the next tt in the
                     // matcher. This is a bug, but #95267 showed that existing programs rely on
@@ -450,7 +459,7 @@ impl TtParser {
                         // Try zero matches of this sequence, by skipping over it.
                         self.cur_mps.push(MatcherPos {
                             idx: idx_first_after,
-                            matches: mp.matches.clone(), // a cheap clone
+                            matches: Lrc::clone(&mp.matches),
                         });
                     }
 
@@ -463,8 +472,8 @@ impl TtParser {
                     // sequence. If that's not possible, `ending_mp` will fail quietly when it is
                     // processed next time around the loop.
                     let ending_mp = MatcherPos {
-                        idx: mp.idx + 1,             // +1 skips the Kleene op
-                        matches: mp.matches.clone(), // a cheap clone
+                        idx: mp.idx + 1, // +1 skips the Kleene op
+                        matches: Lrc::clone(&mp.matches),
                     };
                     self.cur_mps.push(ending_mp);
 
@@ -479,8 +488,8 @@ impl TtParser {
                     // separator yet. Try ending the sequence. If that's not possible, `ending_mp`
                     // will fail quietly when it is processed next time around the loop.
                     let ending_mp = MatcherPos {
-                        idx: mp.idx + 2,             // +2 skips the separator and the Kleene op
-                        matches: mp.matches.clone(), // a cheap clone
+                        idx: mp.idx + 2, // +2 skips the separator and the Kleene op
+                        matches: Lrc::clone(&mp.matches),
                     };
                     self.cur_mps.push(ending_mp);
 
@@ -552,10 +561,11 @@ impl TtParser {
     }
 
     /// Match the token stream from `parser` against `matcher`.
-    pub(super) fn parse_tt(
+    pub(super) fn parse_tt<'matcher, T: Tracker<'matcher>>(
         &mut self,
         parser: &mut Cow<'_, Parser<'_>>,
-        matcher: &[MatcherLoc],
+        matcher: &'matcher [MatcherLoc],
+        track: &mut T,
     ) -> NamedParseResult {
         // A queue of possible matcher positions. We initialize it with the matcher position in
         // which the "dot" is before the first token of the first token tree in `matcher`.
@@ -571,7 +581,8 @@ impl TtParser {
 
             // Process `cur_mps` until either we have finished the input or we need to get some
             // parsing from the black-box parser done.
-            if let Some(res) = self.parse_tt_inner(matcher, &parser.token) {
+            let res = self.parse_tt_inner(matcher, &parser.token, track);
+            if let Some(res) = res {
                 return res;
             }
 
@@ -612,14 +623,14 @@ impl TtParser {
                         // edition-specific matching behavior for non-terminals.
                         let nt = match parser.to_mut().parse_nonterminal(kind) {
                             Err(mut err) => {
-                                err.span_label(
+                                let guarantee = err.span_label(
                                     span,
                                     format!(
                                         "while parsing argument for this `{kind}` macro fragment"
                                     ),
                                 )
                                 .emit();
-                                return ErrorReported;
+                                return ErrorReported(guarantee);
                             }
                             Ok(nt) => nt,
                         };