Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit c87e2bb

Browse files
borsbors
committed
Auto merge of rust-lang#119689 - petrochenkov:markeager, r=<try>
macro_rules: Eagerly mark spans of produced tokens When a declarative macro expands and produces tokens it marks their spans as coming from that specific macro expansion. Marking is a relatively expensive operation - it needs to lock the global hygiene data. Right now marking happens lazily, when a token is actually produced into the output. But that means marking happens 100 times if `$($var)*` expands to a sequence of length 100 (span of `$var` is marked and outputted as a part of the resulting nonterminal token). In this PR I'm trying to perform this marking eagerly and once. - Pros (perf): tokens from sequences are marked once (1 time instead of N). - Cons (perf): tokens that never end up in the output are still marked (1 time instead of 0). - Cons (perf): cloning of the used macro arm's right hand side is required (`src` in `fn transcribe`). - Cons (perf): metavariable tokens of the `tt` kind weren't previously marked but they are marked now (can't tell whether the variable is `tt` this early). However, for rust-lang#119673 we'll need `tt` metavars marked anyway. - Pros (diagnostics): Some erroneous tokens are now correctly reported as coming from a macro expansion.
2 parents 0ee9cfd + 24f88f8 commit c87e2bb

File tree

9 files changed

+122
-38
lines changed

9 files changed

+122
-38
lines changed

‎compiler/rustc_expand/src/mbe.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@ use rustc_span::Span;
1919

2020
/// Contains the sub-token-trees of a "delimited" token tree such as `(a b c)`.
2121
/// The delimiters are not represented explicitly in the `tts` vector.
22-
#[derive(PartialEq, Encodable, Decodable, Debug)]
22+
#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
2323
struct Delimited {
2424
delim: Delimiter,
2525
/// FIXME: #67062 has details about why this is sub-optimal.
2626
tts: Vec<TokenTree>,
2727
}
2828

29-
#[derive(PartialEq, Encodable, Decodable, Debug)]
29+
#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
3030
struct SequenceRepetition {
3131
/// The sequence of token trees
3232
tts: Vec<TokenTree>,
@@ -64,15 +64,15 @@ pub(crate) enum KleeneOp {
6464

6565
/// Similar to `tokenstream::TokenTree`, except that `Sequence`, `MetaVar`, `MetaVarDecl`, and
6666
/// `MetaVarExpr` are "first-class" token trees. Useful for parsing macros.
67-
#[derive(Debug, PartialEq, Encodable, Decodable)]
67+
#[derive(Clone, Debug, PartialEq, Encodable, Decodable)]
6868
enum TokenTree {
6969
Token(Token),
7070
/// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
7171
Delimited(DelimSpan, DelimSpacing, Delimited),
7272
/// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
7373
Sequence(DelimSpan, SequenceRepetition),
7474
/// e.g., `$var`.
75-
MetaVar(Span, Ident),
75+
MetaVar(Span, Ident, Span),
7676
/// e.g., `$var:expr`. Only appears on the LHS.
7777
MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
7878
/// A meta-variable expression inside `${...}`.
@@ -97,7 +97,7 @@ impl TokenTree {
9797
fn span(&self) -> Span {
9898
match *self {
9999
TokenTree::Token(Token { span, .. })
100-
| TokenTree::MetaVar(span, _)
100+
| TokenTree::MetaVar(span, ..)
101101
| TokenTree::MetaVarDecl(span, _, _) => span,
102102
TokenTree::Delimited(span, ..)
103103
| TokenTree::MetaVarExpr(span, _)

‎compiler/rustc_expand/src/mbe/macro_check.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ fn check_binders(
242242
// the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the
243243
// LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon
244244
// MetaVar(fragment) and not as MetaVarDecl(y, fragment).
245-
TokenTree::MetaVar(span, name) => {
245+
TokenTree::MetaVar(span, name, _) => {
246246
if macros.is_empty() {
247247
sess.dcx.span_bug(span, "unexpected MetaVar in lhs");
248248
}
@@ -342,7 +342,7 @@ fn check_occurrences(
342342
TokenTree::MetaVarDecl(span, _name, _kind) => {
343343
sess.dcx.span_bug(span, "unexpected MetaVarDecl in rhs")
344344
}
345-
TokenTree::MetaVar(span, name) => {
345+
TokenTree::MetaVar(span, name, _) => {
346346
let name = MacroRulesNormalizedIdent::new(name);
347347
check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name);
348348
}

‎compiler/rustc_expand/src/mbe/macro_rules.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1378,7 +1378,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13781378
fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
13791379
match tt {
13801380
mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
1381-
mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
1381+
mbe::TokenTree::MetaVar(_, name, _) => format!("${name}"),
13821382
mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
13831383
mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),
13841384
_ => panic!(

‎compiler/rustc_expand/src/mbe/quoted.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ pub(super) fn parse(
5454
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
5555
let tree = parse_tree(tree, &mut trees, parsing_patterns, sess, node_id, features, edition);
5656
match tree {
57-
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
57+
TokenTree::MetaVar(start_sp, ident, _) if parsing_patterns => {
5858
let span = match trees.next() {
5959
Some(&tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
6060
match trees.next() {
@@ -223,7 +223,7 @@ fn parse_tree<'a>(
223223
if ident.name == kw::Crate && !is_raw {
224224
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
225225
} else {
226-
TokenTree::MetaVar(span, ident)
226+
TokenTree::MetaVar(span, ident, ident.span)
227227
}
228228
}
229229

@@ -245,7 +245,8 @@ fn parse_tree<'a>(
245245
let msg =
246246
format!("expected identifier, found `{}`", pprust::token_to_string(token),);
247247
sess.dcx.span_err(token.span, msg);
248-
TokenTree::MetaVar(token.span, Ident::empty())
248+
let ident = Ident::empty();
249+
TokenTree::MetaVar(token.span, ident, ident.span)
249250
}
250251

251252
// There are no more tokens. Just return the `$` we already have.

‎compiler/rustc_expand/src/mbe/transcribe.rs‎

Lines changed: 66 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,55 @@ impl<'a> Iterator for Frame<'a> {
7676
}
7777
}
7878

79+
fn mark_tt(tt: &mut mbe::TokenTree, marker: &mut Marker) {
80+
// Spans that never end up in the output don't need to be marked.
81+
// `_ident`s are metavariable names and need to keep their original spans to resolve correctly
82+
// (they also never end up in the output).
83+
match tt {
84+
mbe::TokenTree::Token(token) => mut_visit::visit_token(token, marker),
85+
mbe::TokenTree::Delimited(dspan, _dspacing, delimited) => {
86+
mut_visit::visit_delim_span(dspan, marker);
87+
mark_delimited(delimited, marker);
88+
}
89+
mbe::TokenTree::Sequence(_dspan, rep) => {
90+
// Sequence delimiter spans never ends up in the output.
91+
mark_sequence_repetition(rep, marker);
92+
}
93+
mbe::TokenTree::MetaVar(span, _ident, marked_span) => {
94+
marker.visit_span(span);
95+
marker.visit_span(marked_span);
96+
}
97+
mbe::TokenTree::MetaVarExpr(dspan, expr) => {
98+
mut_visit::visit_delim_span(dspan, marker);
99+
match expr {
100+
MetaVarExpr::Count(_ident, _depth) => {}
101+
MetaVarExpr::Ignore(_ident) => {}
102+
MetaVarExpr::Index(_depth) | MetaVarExpr::Length(_depth) => {}
103+
}
104+
}
105+
mbe::TokenTree::MetaVarDecl(..) => unreachable!(),
106+
}
107+
}
108+
109+
fn mark_sequence_repetition(rep: &mut mbe::SequenceRepetition, marker: &mut Marker) {
110+
let mbe::SequenceRepetition { tts, separator, kleene, num_captures: _ } = rep;
111+
for tt in tts {
112+
mark_tt(tt, marker);
113+
}
114+
if let Some(sep) = separator {
115+
mut_visit::visit_token(sep, marker);
116+
}
117+
// Kleenee token span never ends up in the output.
118+
let mbe::KleeneToken { span: _, op: _ } = kleene;
119+
}
120+
121+
fn mark_delimited(delimited: &mut mbe::Delimited, marker: &mut Marker) {
122+
let mbe::Delimited { delim: _, tts } = delimited;
123+
for tt in tts {
124+
mark_tt(tt, marker);
125+
}
126+
}
127+
79128
/// This can do Macro-By-Example transcription.
80129
/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the
81130
/// invocation. We are assuming we already know there is a match.
@@ -108,11 +157,15 @@ pub(super) fn transcribe<'a>(
108157
return Ok(TokenStream::default());
109158
}
110159

160+
let mut src = src.clone();
161+
let expn_id = cx.current_expansion.id;
162+
mark_delimited(&mut src, &mut Marker(expn_id, transparency, Default::default()));
163+
111164
// We descend into the RHS (`src`), expanding things as we go. This stack contains the things
112165
// we have yet to expand/are still expanding. We start the stack off with the whole RHS. The
113166
// choice of spacing values doesn't matter.
114167
let mut stack: SmallVec<[Frame<'_>; 1]> =
115-
smallvec![Frame::new(src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))];
168+
smallvec![Frame::new(&src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))];
116169

117170
// As we descend in the RHS, we will need to be able to match nested sequences of matchers.
118171
// `repeats` keeps track of where we are in matching at each level, with the last element being
@@ -132,7 +185,6 @@ pub(super) fn transcribe<'a>(
132185
// again, and we are done transcribing.
133186
let mut result: Vec<TokenTree> = Vec::new();
134187
let mut result_stack = Vec::new();
135-
let mut marker = Marker(cx.current_expansion.id, transparency, Default::default());
136188

137189
loop {
138190
// Look at the last frame on the stack.
@@ -245,10 +297,11 @@ pub(super) fn transcribe<'a>(
245297
}
246298

247299
// Replace the meta-var with the matched token tree from the invocation.
248-
mbe::TokenTree::MetaVar(mut sp, mut original_ident) => {
300+
mbe::TokenTree::MetaVar(sp, original_ident, marked_span) => {
301+
let sp = *sp;
249302
// Find the matched nonterminal from the macro invocation, and use it to replace
250303
// the meta-var.
251-
let ident = MacroRulesNormalizedIdent::new(original_ident);
304+
let ident = MacroRulesNormalizedIdent::new(*original_ident);
252305
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
253306
match cur_matched {
254307
MatchedTokenTree(tt) => {
@@ -260,7 +313,6 @@ pub(super) fn transcribe<'a>(
260313
// Other variables are emitted into the output stream as groups with
261314
// `Delimiter::Invisible` to maintain parsing priorities.
262315
// `Interpolated` is currently used for such groups in rustc parser.
263-
marker.visit_span(&mut sp);
264316
result
265317
.push(TokenTree::token_alone(token::Interpolated(nt.clone()), sp));
266318
}
@@ -272,33 +324,30 @@ pub(super) fn transcribe<'a>(
272324
} else {
273325
// If we aren't able to match the meta-var, we push it back into the result but
274326
// with modified syntax context. (I believe this supports nested macros).
275-
marker.visit_span(&mut sp);
276-
marker.visit_ident(&mut original_ident);
277327
result.push(TokenTree::token_joint_hidden(token::Dollar, sp));
278328
result.push(TokenTree::Token(
279-
Token::from_ast_ident(original_ident),
329+
Token::from_ast_ident(Ident::new(original_ident.name, *marked_span)),
280330
Spacing::Alone,
281331
));
282332
}
283333
}
284334

285335
// Replace meta-variable expressions with the result of their expansion.
286336
mbe::TokenTree::MetaVarExpr(sp, expr) => {
287-
transcribe_metavar_expr(cx, expr, interp, &mut marker, &repeats, &mut result, sp)?;
337+
transcribe_metavar_expr(cx, expr, interp, &repeats, &mut result, sp)?;
288338
}
289339

290340
// If we are entering a new delimiter, we push its contents to the `stack` to be
291341
// processed, and we push all of the currently produced results to the `result_stack`.
292342
// We will produce all of the results of the inside of the `Delimited` and then we will
293343
// jump back out of the Delimited, pop the result_stack and add the new results back to
294344
// the previous results (from outside the Delimited).
295-
mbe::TokenTree::Delimited(mut span, spacing, delimited) => {
296-
mut_visit::visit_delim_span(&mut span, &mut marker);
345+
mbe::TokenTree::Delimited(span, spacing, delimited) => {
297346
stack.push(Frame::Delimited {
298347
tts: &delimited.tts,
299348
delim: delimited.delim,
300349
idx: 0,
301-
span,
350+
span: *span,
302351
spacing: *spacing,
303352
});
304353
result_stack.push(mem::take(&mut result));
@@ -307,10 +356,7 @@ pub(super) fn transcribe<'a>(
307356
// Nothing much to do here. Just push the token to the result, being careful to
308357
// preserve syntax context.
309358
mbe::TokenTree::Token(token) => {
310-
let mut token = token.clone();
311-
mut_visit::visit_token(&mut token, &mut marker);
312-
let tt = TokenTree::Token(token, Spacing::Alone);
313-
result.push(tt);
359+
result.push(TokenTree::Token(token.clone(), Spacing::Alone));
314360
}
315361

316362
// There should be no meta-var declarations in the invocation of a macro.
@@ -475,7 +521,7 @@ fn lockstep_iter_size(
475521
size.with(lockstep_iter_size(tt, interpolations, repeats))
476522
})
477523
}
478-
TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => {
524+
TokenTree::MetaVar(_, name, _) | TokenTree::MetaVarDecl(_, name, _) => {
479525
let name = MacroRulesNormalizedIdent::new(*name);
480526
match lookup_cur_matched(name, interpolations, repeats) {
481527
Some(matched) => match matched {
@@ -620,23 +666,17 @@ fn transcribe_metavar_expr<'a>(
620666
cx: &ExtCtxt<'a>,
621667
expr: &MetaVarExpr,
622668
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
623-
marker: &mut Marker,
624669
repeats: &[(usize, usize)],
625670
result: &mut Vec<TokenTree>,
626671
sp: &DelimSpan,
627672
) -> PResult<'a, ()> {
628-
let mut visited_span = || {
629-
let mut span = sp.entire();
630-
marker.visit_span(&mut span);
631-
span
632-
};
633673
match *expr {
634674
MetaVarExpr::Count(original_ident, depth) => {
635675
let matched = matched_from_ident(cx, original_ident, interp)?;
636676
let count = count_repetitions(cx, depth, matched, repeats, sp)?;
637677
let tt = TokenTree::token_alone(
638678
TokenKind::lit(token::Integer, sym::integer(count), None),
639-
visited_span(),
679+
sp.entire(),
640680
);
641681
result.push(tt);
642682
}
@@ -648,7 +688,7 @@ fn transcribe_metavar_expr<'a>(
648688
Some((index, _)) => {
649689
result.push(TokenTree::token_alone(
650690
TokenKind::lit(token::Integer, sym::integer(*index), None),
651-
visited_span(),
691+
sp.entire(),
652692
));
653693
}
654694
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")),
@@ -657,7 +697,7 @@ fn transcribe_metavar_expr<'a>(
657697
Some((_, length)) => {
658698
result.push(TokenTree::token_alone(
659699
TokenKind::lit(token::Integer, sym::integer(*length), None),
660-
visited_span(),
700+
sp.entire(),
661701
));
662702
}
663703
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")),

‎tests/ui/macros/meta-variable-depth-outside-repeat.stderr‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ error: meta-variable expression `length` with depth parameter must be called ins
33
|
44
LL | ${length(0)}
55
| ^^^^^^^^^^^
6+
...
7+
LL | const _: i32 = metavar!(0);
8+
| ----------- in this macro invocation
9+
|
10+
= note: this error originates in the macro `metavar` (in Nightly builds, run with -Z macro-backtrace for more info)
611

712
error: aborting due to 1 previous error
813

‎tests/ui/macros/rfc-3086-metavar-expr/out-of-bounds-arguments.stderr‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,33 @@ error: depth parameter of meta-variable expression `count` must be less than 4
33
|
44
LL | ${count($foo, 10)},
55
| ^^^^^^^^^^^^^^^^^
6+
...
7+
LL | a!( { [ (a) ] [ (b c) ] } );
8+
| --------------------------- in this macro invocation
9+
|
10+
= note: this error originates in the macro `a` (in Nightly builds, run with -Z macro-backtrace for more info)
611

712
error: depth parameter of meta-variable expression `index` must be less than 3
813
--> $DIR/out-of-bounds-arguments.rs:19:18
914
|
1015
LL | ${index(10)},
1116
| ^^^^^^^^^^^
17+
...
18+
LL | b!( { [ a b ] } );
19+
| ----------------- in this macro invocation
20+
|
21+
= note: this error originates in the macro `b` (in Nightly builds, run with -Z macro-backtrace for more info)
1222

1323
error: depth parameter of meta-variable expression `length` must be less than 2
1424
--> $DIR/out-of-bounds-arguments.rs:32:18
1525
|
1626
LL | ${length(10)}
1727
| ^^^^^^^^^^^^
28+
...
29+
LL | c!({ a });
30+
| --------- in this macro invocation
31+
|
32+
= note: this error originates in the macro `c` (in Nightly builds, run with -Z macro-backtrace for more info)
1833

1934
error: aborting due to 3 previous errors
2035

‎tests/ui/macros/rfc-3086-metavar-expr/syntax-errors.stderr‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,18 +201,33 @@ error: `count` can not be placed inside the inner-most repetition
201201
|
202202
LL | ( $i:ident ) => { ${ count($i) } };
203203
| ^^^^^^^^^^^^^
204+
...
205+
LL | curly__no_rhs_dollar__no_round!(a);
206+
| ---------------------------------- in this macro invocation
207+
|
208+
= note: this error originates in the macro `curly__no_rhs_dollar__no_round` (in Nightly builds, run with -Z macro-backtrace for more info)
204209

205210
error: `count` can not be placed inside the inner-most repetition
206211
--> $DIR/syntax-errors.rs:17:24
207212
|
208213
LL | ( $i:ident ) => { ${ count($i) } };
209214
| ^^^^^^^^^^^^^
215+
...
216+
LL | curly__rhs_dollar__no_round!(a);
217+
| ------------------------------- in this macro invocation
218+
|
219+
= note: this error originates in the macro `curly__rhs_dollar__no_round` (in Nightly builds, run with -Z macro-backtrace for more info)
210220

211221
error: variable 'i' is still repeating at this depth
212222
--> $DIR/syntax-errors.rs:34:36
213223
|
214224
LL | ( $( $i:ident ),* ) => { count($i) };
215225
| ^^
226+
...
227+
LL | no_curly__rhs_dollar__round!(a, b, c);
228+
| ------------------------------------- in this macro invocation
229+
|
230+
= note: this error originates in the macro `no_curly__rhs_dollar__round` (in Nightly builds, run with -Z macro-backtrace for more info)
216231

217232
error: expected expression, found `$`
218233
--> $DIR/syntax-errors.rs:53:9

0 commit comments

Comments
 (0)