Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 52e9fd6

Browse files
borsbors
committed
Auto merge of rust-lang#119689 - petrochenkov:markeager, r=<try>
macro_rules: Eagerly mark spans of produced tokens When a declarative macro expands and produces tokens it marks their spans as coming from that specific macro expansion. Marking is a relatively expensive operation - it needs to lock the global hygiene data. Right now marking happens lazily, when a token is actually produced into the output. But that means marking happens 100 times if `$($var)*` expands to a sequence of length 100 (span of `$var` is marked and outputted as a part of the resulting nonterminal token). In this PR I'm trying to perform this marking eagerly and once. - Pros (perf): tokens from sequences are marked once (1 time instead of N). - Cons (perf): tokens that never end up in the output are still marked (1 time instead of 0). - Cons (perf): cloning of the used macro arm's right hand side is required (`src` in `fn transcribe`). - Cons (perf): metavariable tokens of the `tt` kind weren't previously marked but they are marked now (can't tell whether the variable is `tt` this early). However, for rust-lang#119673 we'll need `tt` metavars marked anyway. - Pros (diagnostics): Some erroneous tokens are now correctly reported as coming from a macro expansion.
2 parents 9522993 + ee50351 commit 52e9fd6

File tree

9 files changed

+121
-38
lines changed

9 files changed

+121
-38
lines changed

‎compiler/rustc_expand/src/mbe.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@ use rustc_span::Span;
1919

2020
/// Contains the sub-token-trees of a "delimited" token tree such as `(a b c)`.
2121
/// The delimiters are not represented explicitly in the `tts` vector.
22-
#[derive(PartialEq, Encodable, Decodable, Debug)]
22+
#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
2323
struct Delimited {
2424
delim: Delimiter,
2525
/// FIXME: #67062 has details about why this is sub-optimal.
2626
tts: Vec<TokenTree>,
2727
}
2828

29-
#[derive(PartialEq, Encodable, Decodable, Debug)]
29+
#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
3030
struct SequenceRepetition {
3131
/// The sequence of token trees
3232
tts: Vec<TokenTree>,
@@ -64,15 +64,15 @@ pub(crate) enum KleeneOp {
6464

6565
/// Similar to `tokenstream::TokenTree`, except that `Sequence`, `MetaVar`, `MetaVarDecl`, and
6666
/// `MetaVarExpr` are "first-class" token trees. Useful for parsing macros.
67-
#[derive(Debug, PartialEq, Encodable, Decodable)]
67+
#[derive(Clone, Debug, PartialEq, Encodable, Decodable)]
6868
enum TokenTree {
6969
Token(Token),
7070
/// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
7171
Delimited(DelimSpan, DelimSpacing, Delimited),
7272
/// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
7373
Sequence(DelimSpan, SequenceRepetition),
7474
/// e.g., `$var`.
75-
MetaVar(Span, Ident),
75+
MetaVar(Span, Ident, Span),
7676
/// e.g., `$var:expr`. Only appears on the LHS.
7777
MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
7878
/// A meta-variable expression inside `${...}`.
@@ -97,7 +97,7 @@ impl TokenTree {
9797
fn span(&self) -> Span {
9898
match *self {
9999
TokenTree::Token(Token { span, .. })
100-
| TokenTree::MetaVar(span, _)
100+
| TokenTree::MetaVar(span, ..)
101101
| TokenTree::MetaVarDecl(span, _, _) => span,
102102
TokenTree::Delimited(span, ..)
103103
| TokenTree::MetaVarExpr(span, _)

‎compiler/rustc_expand/src/mbe/macro_check.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ fn check_binders(
242242
// the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the
243243
// LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon
244244
// MetaVar(fragment) and not as MetaVarDecl(y, fragment).
245-
TokenTree::MetaVar(span, name) => {
245+
TokenTree::MetaVar(span, name, _) => {
246246
if macros.is_empty() {
247247
sess.dcx.span_bug(span, "unexpected MetaVar in lhs");
248248
}
@@ -342,7 +342,7 @@ fn check_occurrences(
342342
TokenTree::MetaVarDecl(span, _name, _kind) => {
343343
sess.dcx.span_bug(span, "unexpected MetaVarDecl in rhs")
344344
}
345-
TokenTree::MetaVar(span, name) => {
345+
TokenTree::MetaVar(span, name, _) => {
346346
let name = MacroRulesNormalizedIdent::new(name);
347347
check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name);
348348
}

‎compiler/rustc_expand/src/mbe/macro_rules.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1378,7 +1378,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13781378
fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
13791379
match tt {
13801380
mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
1381-
mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
1381+
mbe::TokenTree::MetaVar(_, name, _) => format!("${name}"),
13821382
mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
13831383
mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),
13841384
_ => panic!(

‎compiler/rustc_expand/src/mbe/quoted.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ pub(super) fn parse(
5454
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
5555
let tree = parse_tree(tree, &mut trees, parsing_patterns, sess, node_id, features, edition);
5656
match tree {
57-
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
57+
TokenTree::MetaVar(start_sp, ident, _) if parsing_patterns => {
5858
let span = match trees.next() {
5959
Some(&tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
6060
match trees.next() {
@@ -223,7 +223,7 @@ fn parse_tree<'a>(
223223
if ident.name == kw::Crate && !is_raw {
224224
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
225225
} else {
226-
TokenTree::MetaVar(span, ident)
226+
TokenTree::MetaVar(span, ident, ident.span)
227227
}
228228
}
229229

@@ -245,7 +245,8 @@ fn parse_tree<'a>(
245245
let msg =
246246
format!("expected identifier, found `{}`", pprust::token_to_string(token),);
247247
sess.dcx.span_err(token.span, msg);
248-
TokenTree::MetaVar(token.span, Ident::empty())
248+
let ident = Ident::empty();
249+
TokenTree::MetaVar(token.span, ident, ident.span)
249250
}
250251

251252
// There are no more tokens. Just return the `$` we already have.

‎compiler/rustc_expand/src/mbe/transcribe.rs‎

Lines changed: 65 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,55 @@ impl<'a> Iterator for Frame<'a> {
6767
}
6868
}
6969

70+
fn mark_tt(tt: &mut mbe::TokenTree, marker: &mut Marker) {
71+
// Spans that never end up in the output don't need to be marked.
72+
// `_ident`s are metavariable names and need to keep their original spans to resolve correctly
73+
// (they also never end up in the output).
74+
match tt {
75+
mbe::TokenTree::Token(token) => mut_visit::visit_token(token, marker),
76+
mbe::TokenTree::Delimited(dspan, _dspacing, delimited) => {
77+
mut_visit::visit_delim_span(dspan, marker);
78+
mark_delimited(delimited, marker);
79+
}
80+
mbe::TokenTree::Sequence(_dspan, rep) => {
81+
// Sequence delimiter spans never ends up in the output.
82+
mark_sequence_repetition(rep, marker);
83+
}
84+
mbe::TokenTree::MetaVar(span, _ident, marked_span) => {
85+
marker.visit_span(span);
86+
marker.visit_span(marked_span);
87+
}
88+
mbe::TokenTree::MetaVarExpr(dspan, expr) => {
89+
mut_visit::visit_delim_span(dspan, marker);
90+
match expr {
91+
MetaVarExpr::Count(_ident, _depth) => {}
92+
MetaVarExpr::Ignore(_ident) => {}
93+
MetaVarExpr::Index(_depth) | MetaVarExpr::Length(_depth) => {}
94+
}
95+
}
96+
mbe::TokenTree::MetaVarDecl(..) => unreachable!(),
97+
}
98+
}
99+
100+
fn mark_sequence_repetition(rep: &mut mbe::SequenceRepetition, marker: &mut Marker) {
101+
let mbe::SequenceRepetition { tts, separator, kleene, num_captures: _ } = rep;
102+
for tt in tts {
103+
mark_tt(tt, marker);
104+
}
105+
if let Some(sep) = separator {
106+
mut_visit::visit_token(sep, marker);
107+
}
108+
// Kleenee token span never ends up in the output.
109+
let mbe::KleeneToken { span: _, op: _ } = kleene;
110+
}
111+
112+
fn mark_delimited(delimited: &mut mbe::Delimited, marker: &mut Marker) {
113+
let mbe::Delimited { delim: _, tts } = delimited;
114+
for tt in tts {
115+
mark_tt(tt, marker);
116+
}
117+
}
118+
70119
/// This can do Macro-By-Example transcription.
71120
/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the
72121
/// invocation. We are assuming we already know there is a match.
@@ -99,11 +148,14 @@ pub(super) fn transcribe<'a>(
99148
return Ok(TokenStream::default());
100149
}
101150

151+
let mut src = src.clone();
152+
mark_delimited(&mut src, &mut Marker(cx.current_expansion.id, transparency));
153+
102154
// We descend into the RHS (`src`), expanding things as we go. This stack contains the things
103155
// we have yet to expand/are still expanding. We start the stack off with the whole RHS. The
104156
// choice of spacing values doesn't matter.
105157
let mut stack: SmallVec<[Frame<'_>; 1]> =
106-
smallvec![Frame::new(src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))];
158+
smallvec![Frame::new(&src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))];
107159

108160
// As we descend in the RHS, we will need to be able to match nested sequences of matchers.
109161
// `repeats` keeps track of where we are in matching at each level, with the last element being
@@ -123,7 +175,6 @@ pub(super) fn transcribe<'a>(
123175
// again, and we are done transcribing.
124176
let mut result: Vec<TokenTree> = Vec::new();
125177
let mut result_stack = Vec::new();
126-
let mut marker = Marker(cx.current_expansion.id, transparency);
127178

128179
loop {
129180
// Look at the last frame on the stack.
@@ -236,10 +287,11 @@ pub(super) fn transcribe<'a>(
236287
}
237288

238289
// Replace the meta-var with the matched token tree from the invocation.
239-
mbe::TokenTree::MetaVar(mut sp, mut original_ident) => {
290+
mbe::TokenTree::MetaVar(sp, original_ident, marked_span) => {
291+
let sp = *sp;
240292
// Find the matched nonterminal from the macro invocation, and use it to replace
241293
// the meta-var.
242-
let ident = MacroRulesNormalizedIdent::new(original_ident);
294+
let ident = MacroRulesNormalizedIdent::new(*original_ident);
243295
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
244296
match cur_matched {
245297
MatchedTokenTree(tt) => {
@@ -251,7 +303,6 @@ pub(super) fn transcribe<'a>(
251303
// Other variables are emitted into the output stream as groups with
252304
// `Delimiter::Invisible` to maintain parsing priorities.
253305
// `Interpolated` is currently used for such groups in rustc parser.
254-
marker.visit_span(&mut sp);
255306
result
256307
.push(TokenTree::token_alone(token::Interpolated(nt.clone()), sp));
257308
}
@@ -263,33 +314,30 @@ pub(super) fn transcribe<'a>(
263314
} else {
264315
// If we aren't able to match the meta-var, we push it back into the result but
265316
// with modified syntax context. (I believe this supports nested macros).
266-
marker.visit_span(&mut sp);
267-
marker.visit_ident(&mut original_ident);
268317
result.push(TokenTree::token_joint_hidden(token::Dollar, sp));
269318
result.push(TokenTree::Token(
270-
Token::from_ast_ident(original_ident),
319+
Token::from_ast_ident(Ident::new(original_ident.name, *marked_span)),
271320
Spacing::Alone,
272321
));
273322
}
274323
}
275324

276325
// Replace meta-variable expressions with the result of their expansion.
277326
mbe::TokenTree::MetaVarExpr(sp, expr) => {
278-
transcribe_metavar_expr(cx, expr, interp, &mut marker, &repeats, &mut result, sp)?;
327+
transcribe_metavar_expr(cx, expr, interp, &repeats, &mut result, sp)?;
279328
}
280329

281330
// If we are entering a new delimiter, we push its contents to the `stack` to be
282331
// processed, and we push all of the currently produced results to the `result_stack`.
283332
// We will produce all of the results of the inside of the `Delimited` and then we will
284333
// jump back out of the Delimited, pop the result_stack and add the new results back to
285334
// the previous results (from outside the Delimited).
286-
mbe::TokenTree::Delimited(mut span, spacing, delimited) => {
287-
mut_visit::visit_delim_span(&mut span, &mut marker);
335+
mbe::TokenTree::Delimited(span, spacing, delimited) => {
288336
stack.push(Frame::Delimited {
289337
tts: &delimited.tts,
290338
delim: delimited.delim,
291339
idx: 0,
292-
span,
340+
span: *span,
293341
spacing: *spacing,
294342
});
295343
result_stack.push(mem::take(&mut result));
@@ -298,10 +346,7 @@ pub(super) fn transcribe<'a>(
298346
// Nothing much to do here. Just push the token to the result, being careful to
299347
// preserve syntax context.
300348
mbe::TokenTree::Token(token) => {
301-
let mut token = token.clone();
302-
mut_visit::visit_token(&mut token, &mut marker);
303-
let tt = TokenTree::Token(token, Spacing::Alone);
304-
result.push(tt);
349+
result.push(TokenTree::Token(token.clone(), Spacing::Alone));
305350
}
306351

307352
// There should be no meta-var declarations in the invocation of a macro.
@@ -466,7 +511,7 @@ fn lockstep_iter_size(
466511
size.with(lockstep_iter_size(tt, interpolations, repeats))
467512
})
468513
}
469-
TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => {
514+
TokenTree::MetaVar(_, name, _) | TokenTree::MetaVarDecl(_, name, _) => {
470515
let name = MacroRulesNormalizedIdent::new(*name);
471516
match lookup_cur_matched(name, interpolations, repeats) {
472517
Some(matched) => match matched {
@@ -611,23 +656,17 @@ fn transcribe_metavar_expr<'a>(
611656
cx: &ExtCtxt<'a>,
612657
expr: &MetaVarExpr,
613658
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
614-
marker: &mut Marker,
615659
repeats: &[(usize, usize)],
616660
result: &mut Vec<TokenTree>,
617661
sp: &DelimSpan,
618662
) -> PResult<'a, ()> {
619-
let mut visited_span = || {
620-
let mut span = sp.entire();
621-
marker.visit_span(&mut span);
622-
span
623-
};
624663
match *expr {
625664
MetaVarExpr::Count(original_ident, depth) => {
626665
let matched = matched_from_ident(cx, original_ident, interp)?;
627666
let count = count_repetitions(cx, depth, matched, repeats, sp)?;
628667
let tt = TokenTree::token_alone(
629668
TokenKind::lit(token::Integer, sym::integer(count), None),
630-
visited_span(),
669+
sp.entire(),
631670
);
632671
result.push(tt);
633672
}
@@ -639,7 +678,7 @@ fn transcribe_metavar_expr<'a>(
639678
Some((index, _)) => {
640679
result.push(TokenTree::token_alone(
641680
TokenKind::lit(token::Integer, sym::integer(*index), None),
642-
visited_span(),
681+
sp.entire(),
643682
));
644683
}
645684
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")),
@@ -648,7 +687,7 @@ fn transcribe_metavar_expr<'a>(
648687
Some((_, length)) => {
649688
result.push(TokenTree::token_alone(
650689
TokenKind::lit(token::Integer, sym::integer(*length), None),
651-
visited_span(),
690+
sp.entire(),
652691
));
653692
}
654693
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")),

‎tests/ui/macros/meta-variable-depth-outside-repeat.stderr‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ error: meta-variable expression `length` with depth parameter must be called ins
33
|
44
LL | ${length(0)}
55
| ^^^^^^^^^^^
6+
...
7+
LL | const _: i32 = metavar!(0);
8+
| ----------- in this macro invocation
9+
|
10+
= note: this error originates in the macro `metavar` (in Nightly builds, run with -Z macro-backtrace for more info)
611

712
error: aborting due to 1 previous error
813

‎tests/ui/macros/rfc-3086-metavar-expr/out-of-bounds-arguments.stderr‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,33 @@ error: depth parameter of meta-variable expression `count` must be less than 4
33
|
44
LL | ${count($foo, 10)},
55
| ^^^^^^^^^^^^^^^^^
6+
...
7+
LL | a!( { [ (a) ] [ (b c) ] } );
8+
| --------------------------- in this macro invocation
9+
|
10+
= note: this error originates in the macro `a` (in Nightly builds, run with -Z macro-backtrace for more info)
611

712
error: depth parameter of meta-variable expression `index` must be less than 3
813
--> $DIR/out-of-bounds-arguments.rs:19:18
914
|
1015
LL | ${index(10)},
1116
| ^^^^^^^^^^^
17+
...
18+
LL | b!( { [ a b ] } );
19+
| ----------------- in this macro invocation
20+
|
21+
= note: this error originates in the macro `b` (in Nightly builds, run with -Z macro-backtrace for more info)
1222

1323
error: depth parameter of meta-variable expression `length` must be less than 2
1424
--> $DIR/out-of-bounds-arguments.rs:32:18
1525
|
1626
LL | ${length(10)}
1727
| ^^^^^^^^^^^^
28+
...
29+
LL | c!({ a });
30+
| --------- in this macro invocation
31+
|
32+
= note: this error originates in the macro `c` (in Nightly builds, run with -Z macro-backtrace for more info)
1833

1934
error: aborting due to 3 previous errors
2035

‎tests/ui/macros/rfc-3086-metavar-expr/syntax-errors.stderr‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,18 +201,33 @@ error: `count` can not be placed inside the inner-most repetition
201201
|
202202
LL | ( $i:ident ) => { ${ count($i) } };
203203
| ^^^^^^^^^^^^^
204+
...
205+
LL | curly__no_rhs_dollar__no_round!(a);
206+
| ---------------------------------- in this macro invocation
207+
|
208+
= note: this error originates in the macro `curly__no_rhs_dollar__no_round` (in Nightly builds, run with -Z macro-backtrace for more info)
204209

205210
error: `count` can not be placed inside the inner-most repetition
206211
--> $DIR/syntax-errors.rs:17:24
207212
|
208213
LL | ( $i:ident ) => { ${ count($i) } };
209214
| ^^^^^^^^^^^^^
215+
...
216+
LL | curly__rhs_dollar__no_round!(a);
217+
| ------------------------------- in this macro invocation
218+
|
219+
= note: this error originates in the macro `curly__rhs_dollar__no_round` (in Nightly builds, run with -Z macro-backtrace for more info)
210220

211221
error: variable 'i' is still repeating at this depth
212222
--> $DIR/syntax-errors.rs:34:36
213223
|
214224
LL | ( $( $i:ident ),* ) => { count($i) };
215225
| ^^
226+
...
227+
LL | no_curly__rhs_dollar__round!(a, b, c);
228+
| ------------------------------------- in this macro invocation
229+
|
230+
= note: this error originates in the macro `no_curly__rhs_dollar__round` (in Nightly builds, run with -Z macro-backtrace for more info)
216231

217232
error: expected expression, found `$`
218233
--> $DIR/syntax-errors.rs:53:9

0 commit comments

Comments
 (0)