Retry failed macro matching for diagnostics

compiler/rustc_expand/src/mbe.rs

@@ -52,7 +52,7 @@ impl KleeneToken {
 /// A Kleene-style [repetition operator](https://en.wikipedia.org/wiki/Kleene_star)
 /// for token sequences.
 #[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy)]
-enum KleeneOp {
+pub(crate) enum KleeneOp {
     /// Kleene star (`*`) for zero or more repetitions
     ZeroOrMore,
     /// Kleene plus (`+`) for one or more repetitions

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -70,20 +70,20 @@
 //! eof: [a $( a )* a b ·]
 //! ```
 
+use rustc_errors::ErrorGuaranteed;
 pub(crate) use NamedMatch::*;
 pub(crate) use ParseResult::*;
 
-use crate::mbe::{KleeneOp, TokenTree};
+use crate::mbe::{macro_rules::Tracker, KleeneOp, TokenTree};
 
 use rustc_ast::token::{self, DocComment, Nonterminal, NonterminalKind, Token};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Lrc;
 use rustc_lint_defs::pluralize;
 use rustc_parse::parser::{NtOrTt, Parser};
+use rustc_span::symbol::Ident;
 use rustc_span::symbol::MacroRulesNormalizedIdent;
 use rustc_span::Span;
-
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::sync::Lrc;
-use rustc_span::symbol::Ident;
 use std::borrow::Cow;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
 
@@ -96,7 +96,8 @@ use std::collections::hash_map::Entry::{Occupied, Vacant};
 ///
 /// This means a matcher can be represented by `&[MatcherLoc]`, and traversal mostly involves
 /// simply incrementing the current matcher position index by one.
-pub(super) enum MatcherLoc {
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) enum MatcherLoc {
     Token {
         token: Token,
     },
@@ -270,7 +271,7 @@ pub(crate) enum ParseResult<T> {
     Failure(Token, &'static str),
     /// Fatal error (malformed macro?). Abort compilation.
     Error(rustc_span::Span, String),
-    ErrorReported,
+    ErrorReported(ErrorGuaranteed),
 }
 
 /// A `ParseResult` where the `Success` variant contains a mapping of
@@ -400,17 +401,21 @@ impl TtParser {
     ///
     /// `Some(result)` if everything is finished, `None` otherwise. Note that matches are kept
     /// track of through the mps generated.
-    fn parse_tt_inner(
+    fn parse_tt_inner<'matcher, T: Tracker<'matcher>>(
         &mut self,
-        matcher: &[MatcherLoc],
+        matcher: &'matcher [MatcherLoc],
         token: &Token,
+        track: &mut T,
     ) -> Option<NamedParseResult> {
         // Matcher positions that would be valid if the macro invocation was over now. Only
         // modified if `token == Eof`.
         let mut eof_mps = EofMatcherPositions::None;
 
         while let Some(mut mp) = self.cur_mps.pop() {
-            match &matcher[mp.idx] {
+            let matcher_loc = &matcher[mp.idx];
+            track.before_match_loc(self, matcher_loc);
+
+            match matcher_loc {
                 MatcherLoc::Token { token: t } => {
                     // If it's a doc comment, we just ignore it and move on to the next tt in the
                     // matcher. This is a bug, but #95267 showed that existing programs rely on
@@ -450,7 +455,7 @@ impl TtParser {
                         // Try zero matches of this sequence, by skipping over it.
                         self.cur_mps.push(MatcherPos {
                             idx: idx_first_after,
-                            matches: mp.matches.clone(), // a cheap clone
+                            matches: Lrc::clone(&mp.matches),
                         });
                     }
 
@@ -463,8 +468,8 @@ impl TtParser {
                     // sequence. If that's not possible, `ending_mp` will fail quietly when it is
                     // processed next time around the loop.
                     let ending_mp = MatcherPos {
-                        idx: mp.idx + 1,             // +1 skips the Kleene op
-                        matches: mp.matches.clone(), // a cheap clone
+                        idx: mp.idx + 1, // +1 skips the Kleene op
+                        matches: Lrc::clone(&mp.matches),
                     };
                     self.cur_mps.push(ending_mp);
 
@@ -479,8 +484,8 @@ impl TtParser {
                     // separator yet. Try ending the sequence. If that's not possible, `ending_mp`
                     // will fail quietly when it is processed next time around the loop.
                     let ending_mp = MatcherPos {
-                        idx: mp.idx + 2,             // +2 skips the separator and the Kleene op
-                        matches: mp.matches.clone(), // a cheap clone
+                        idx: mp.idx + 2, // +2 skips the separator and the Kleene op
+                        matches: Lrc::clone(&mp.matches),
                     };
                     self.cur_mps.push(ending_mp);
 
@@ -552,10 +557,11 @@ impl TtParser {
     }
 
     /// Match the token stream from `parser` against `matcher`.
-    pub(super) fn parse_tt(
+    pub(super) fn parse_tt<'matcher, T: Tracker<'matcher>>(
         &mut self,
         parser: &mut Cow<'_, Parser<'_>>,
-        matcher: &[MatcherLoc],
+        matcher: &'matcher [MatcherLoc],
+        track: &mut T,
     ) -> NamedParseResult {
         // A queue of possible matcher positions. We initialize it with the matcher position in
         // which the "dot" is before the first token of the first token tree in `matcher`.
@@ -571,7 +577,8 @@ impl TtParser {
 
             // Process `cur_mps` until either we have finished the input or we need to get some
             // parsing from the black-box parser done.
-            if let Some(res) = self.parse_tt_inner(matcher, &parser.token) {
+            let res = self.parse_tt_inner(matcher, &parser.token, track);
+            if let Some(res) = res {
                 return res;
             }
 
@@ -612,14 +619,14 @@ impl TtParser {
                         // edition-specific matching behavior for non-terminals.
                         let nt = match parser.to_mut().parse_nonterminal(kind) {
                             Err(mut err) => {
-                                err.span_label(
+                                let guarantee = err.span_label(
                                     span,
                                     format!(
                                         "while parsing argument for this `{kind}` macro fragment"
                                     ),
                                 )
                                 .emit();
-                                return ErrorReported;
+                                return ErrorReported(guarantee);
                             }
                             Ok(nt) => nt,
                         };

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -33,6 +33,8 @@ use std::borrow::Cow;
 use std::collections::hash_map::Entry;
 use std::{mem, slice};
 
+use super::macro_parser::NamedParseResult;
+
 pub(crate) struct ParserAnyMacro<'a> {
     parser: Parser<'a>,
 
@@ -205,6 +207,29 @@ fn trace_macros_note(cx_expansions: &mut FxIndexMap<Span, Vec<String>>, sp: Span
     cx_expansions.entry(sp).or_default().push(message);
 }
 
+pub(super) trait Tracker<'matcher> {
+    /// This is called before trying to match next MatcherLoc on the current token
+    fn before_match_loc(&mut self, parser: &TtParser, matcher: &'matcher MatcherLoc);
+
+    /// This is called after an arm has been parsed, either successfully or unsuccessfully. When this is called,
+    /// `before_match_loc` was called at least once (with a `MatcherLoc::Eof`)
+    fn after_arm(&mut self, result: &NamedParseResult);
+
+    /// For tracing
+    fn description() -> &'static str;
+}
+
+/// A noop tracker that is used in the hot path of the expansion, has zero overhead thanks to monomorphization
+struct NoopTracker;
+
+impl<'matcher> Tracker<'matcher> for NoopTracker {
+    fn before_match_loc(&mut self, _: &TtParser, _: &'matcher MatcherLoc) {}
+    fn after_arm(&mut self, _: &NamedParseResult) {}
+    fn description() -> &'static str {
+        "none"
+    }
+}
+
 /// Expands the rules based macro defined by `lhses` and `rhses` for a given
 /// input `arg`.
 fn expand_macro<'cx>(
@@ -262,7 +287,7 @@ fn expand_macro<'cx>(
         // are not recorded. On the first `Success(..)`ful matcher, the spans are merged.
         let mut gated_spans_snapshot = mem::take(&mut *sess.gated_spans.spans.borrow_mut());
 
-        match tt_parser.parse_tt(&mut Cow::Borrowed(&parser), lhs) {
+        match tt_parser.parse_tt(&mut Cow::Borrowed(&parser), lhs, &mut NoopTracker) {
             Success(named_matches) => {
                 // The matcher was `Success(..)`ful.
                 // Merge the gated spans from parsing the matcher with the pre-existing ones.
@@ -333,7 +358,7 @@ fn expand_macro<'cx>(
                 cx.struct_span_err(span, &msg).emit();
                 return DummyResult::any(span);
             }
-            ErrorReported => return DummyResult::any(sp),
+            ErrorReported(_) => return DummyResult::any(sp),
         }
 
         // The matcher was not `Success(..)`ful.
@@ -354,7 +379,7 @@ fn expand_macro<'cx>(
     if let Some((arg, comma_span)) = arg.add_comma() {
         for lhs in lhses {
             let parser = parser_from_cx(sess, arg.clone());
-            if let Success(_) = tt_parser.parse_tt(&mut Cow::Borrowed(&parser), lhs) {
+            if let Success(_) = tt_parser.parse_tt(&mut Cow::Borrowed(&parser), lhs, &mut NoopTracker) {
                 if comma_span.is_dummy() {
                     err.note("you might be missing a comma");
                 } else {
@@ -452,7 +477,7 @@ pub fn compile_declarative_macro(
     let parser = Parser::new(&sess.parse_sess, body, true, rustc_parse::MACRO_ARGUMENTS);
     let mut tt_parser =
         TtParser::new(Ident::with_dummy_span(if macro_rules { kw::MacroRules } else { kw::Macro }));
-    let argument_map = match tt_parser.parse_tt(&mut Cow::Borrowed(&parser), &argument_gram) {
+    let argument_map = match tt_parser.parse_tt(&mut Cow::Borrowed(&parser), &argument_gram, &mut NoopTracker) {
         Success(m) => m,
         Failure(token, msg) => {
             let s = parse_failure_msg(&token);
@@ -470,7 +495,7 @@ pub fn compile_declarative_macro(
                 .emit();
             return dummy_syn_ext();
         }
-        ErrorReported => {
+        ErrorReported(_) => {
             return dummy_syn_ext();
         }
     };