Skip to content

Commit ba6275b

Browse files
borsbors
committed
Auto merge of #82608 - Aaron1011:feature/final-preexp-tts, r=petrochenkov
Implement token-based handling of attributes during expansion This PR modifies the macro expansion infrastructure to handle attributes in a fully token-based manner. As a result: * Derives macros no longer lose spans when their input is modified by eager cfg-expansion. This is accomplished by performing eager cfg-expansion on the token stream that we pass to the derive proc-macro * Inner attributes now preserve spans in all cases, including when we have multiple inner attributes in a row. This is accomplished through the following changes: * New structs `AttrAnnotatedTokenStream` and `AttrAnnotatedTokenTree` are introduced. These are very similar to a normal `TokenTree`, but they also track the position of attributes and attribute targets within the stream. They are built when we collect tokens during parsing. An `AttrAnnotatedTokenStream` is converted to a regular `TokenStream` when we invoke a macro. * Token capturing and `LazyTokenStream` are modified to work with `AttrAnnotatedTokenStream`. A new `ReplaceRange` type is introduced, which is created during the parsing of a nested AST node to make the 'outer' AST node aware of the attributes and attribute target stored deeper in the token stream. * When we need to perform eager cfg-expansion (either due to `#[derive]` or `#[cfg_eval]`), we tokenize and reparse our target, capturing additional information about the locations of `#[cfg]` and `#[cfg_attr]` attributes at any depth within the target. This is a performance optimization, allowing us to perform less work in the typical case where captured tokens never have eager cfg-expansion run.
2 parents 28b948f + a93c4f0 commit ba6275b

33 files changed

+2041
-1187
lines changed

‎compiler/rustc_ast/src/ast_like.rs

+95-7
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,32 @@
11
use super::ptr::P;
2+
use super::token::Nonterminal;
23
use super::tokenstream::LazyTokenStream;
34
use super::{Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
4-
use super::{AssocItem, Expr, ForeignItem, Item, Local};
5+
use super::{AssocItem, Expr, ForeignItem, Item, Local, MacCallStmt};
56
use super::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
67
use super::{AttrVec, Attribute, Stmt, StmtKind};
78

9+
use std::fmt::Debug;
10+
811
/// An `AstLike` represents an AST node (or some wrapper around
912
/// and AST node) which stores some combination of attributes
1013
/// and tokens.
11-
pub trait AstLike: Sized {
14+
pub trait AstLike: Sized + Debug {
15+
/// This is `true` if this `AstLike` might support 'custom' (proc-macro) inner
16+
/// attributes. Attributes like `#![cfg]` and `#![cfg_attr]` are not
17+
/// considered 'custom' attributes
18+
///
19+
/// If this is `false`, then this `AstLike` definitely does
20+
/// not support 'custom' inner attributes, which enables some optimizations
21+
/// during token collection.
22+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool;
1223
fn attrs(&self) -> &[Attribute];
1324
fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));
1425
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
1526
}
1627

1728
impl<T: AstLike + 'static> AstLike for P<T> {
29+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
1830
fn attrs(&self) -> &[Attribute] {
1931
(**self).attrs()
2032
}
@@ -26,6 +38,55 @@ impl<T: AstLike + 'static> AstLike for P<T> {
2638
}
2739
}
2840

41+
impl AstLike for crate::token::Nonterminal {
42+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
43+
fn attrs(&self) -> &[Attribute] {
44+
match self {
45+
Nonterminal::NtItem(item) => item.attrs(),
46+
Nonterminal::NtStmt(stmt) => stmt.attrs(),
47+
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.attrs(),
48+
Nonterminal::NtPat(_)
49+
| Nonterminal::NtTy(_)
50+
| Nonterminal::NtMeta(_)
51+
| Nonterminal::NtPath(_)
52+
| Nonterminal::NtVis(_)
53+
| Nonterminal::NtTT(_)
54+
| Nonterminal::NtBlock(_)
55+
| Nonterminal::NtIdent(..)
56+
| Nonterminal::NtLifetime(_) => &[],
57+
}
58+
}
59+
fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
60+
match self {
61+
Nonterminal::NtItem(item) => item.visit_attrs(f),
62+
Nonterminal::NtStmt(stmt) => stmt.visit_attrs(f),
63+
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.visit_attrs(f),
64+
Nonterminal::NtPat(_)
65+
| Nonterminal::NtTy(_)
66+
| Nonterminal::NtMeta(_)
67+
| Nonterminal::NtPath(_)
68+
| Nonterminal::NtVis(_)
69+
| Nonterminal::NtTT(_)
70+
| Nonterminal::NtBlock(_)
71+
| Nonterminal::NtIdent(..)
72+
| Nonterminal::NtLifetime(_) => {}
73+
}
74+
}
75+
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
76+
match self {
77+
Nonterminal::NtItem(item) => item.tokens_mut(),
78+
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
79+
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
80+
Nonterminal::NtPat(pat) => pat.tokens_mut(),
81+
Nonterminal::NtTy(ty) => ty.tokens_mut(),
82+
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
83+
Nonterminal::NtPath(path) => path.tokens_mut(),
84+
Nonterminal::NtVis(vis) => vis.tokens_mut(),
85+
_ => panic!("Called tokens_mut on {:?}", self),
86+
}
87+
}
88+
}
89+
2990
fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
3091
crate::mut_visit::visit_clobber(attrs, |attrs| {
3192
let mut vec = attrs.into();
@@ -35,6 +96,10 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
3596
}
3697

3798
impl AstLike for StmtKind {
99+
// This might be an `StmtKind::Item`, which contains
100+
// an item that supports inner attrs
101+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
102+
38103
fn attrs(&self) -> &[Attribute] {
39104
match self {
40105
StmtKind::Local(local) => local.attrs(),
@@ -66,6 +131,8 @@ impl AstLike for StmtKind {
66131
}
67132

68133
impl AstLike for Stmt {
134+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = StmtKind::SUPPORTS_CUSTOM_INNER_ATTRS;
135+
69136
fn attrs(&self) -> &[Attribute] {
70137
self.kind.attrs()
71138
}
@@ -79,6 +146,8 @@ impl AstLike for Stmt {
79146
}
80147

81148
impl AstLike for Attribute {
149+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
150+
82151
fn attrs(&self) -> &[Attribute] {
83152
&[]
84153
}
@@ -94,6 +163,8 @@ impl AstLike for Attribute {
94163
}
95164

96165
impl<T: AstLike> AstLike for Option<T> {
166+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
167+
97168
fn attrs(&self) -> &[Attribute] {
98169
self.as_ref().map(|inner| inner.attrs()).unwrap_or(&[])
99170
}
@@ -127,8 +198,13 @@ impl VecOrAttrVec for AttrVec {
127198
}
128199

129200
macro_rules! derive_has_tokens_and_attrs {
130-
($($ty:path),*) => { $(
201+
(
202+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs:literal;
203+
$($ty:path),*
204+
) => { $(
131205
impl AstLike for $ty {
206+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs;
207+
132208
fn attrs(&self) -> &[Attribute] {
133209
&self.attrs
134210
}
@@ -140,13 +216,16 @@ macro_rules! derive_has_tokens_and_attrs {
140216
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
141217
Some(&mut self.tokens)
142218
}
219+
143220
}
144221
)* }
145222
}
146223

147224
macro_rules! derive_has_attrs_no_tokens {
148225
($($ty:path),*) => { $(
149226
impl AstLike for $ty {
227+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
228+
150229
fn attrs(&self) -> &[Attribute] {
151230
&self.attrs
152231
}
@@ -165,23 +244,32 @@ macro_rules! derive_has_attrs_no_tokens {
165244
macro_rules! derive_has_tokens_no_attrs {
166245
($($ty:path),*) => { $(
167246
impl AstLike for $ty {
247+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
248+
168249
fn attrs(&self) -> &[Attribute] {
169250
&[]
170251
}
171252

172253
fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
173-
174254
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
175255
Some(&mut self.tokens)
176256
}
177257
}
178258
)* }
179259
}
180260

181-
// These AST nodes support both inert and active
182-
// attributes, so they also have tokens.
261+
// These ast nodes support both active and inert attributes,
262+
// so they have tokens collected to pass to proc macros
263+
derive_has_tokens_and_attrs! {
264+
// Both `Item` and `AssocItem` can have bodies, which
265+
// can contain inner attributes
266+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
267+
Item, AssocItem, ForeignItem
268+
}
269+
183270
derive_has_tokens_and_attrs! {
184-
Item, Expr, Local, AssocItem, ForeignItem
271+
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
272+
Local, MacCallStmt, Expr
185273
}
186274

187275
// These ast nodes only support inert attributes, so they don't

‎compiler/rustc_ast/src/attr/mod.rs

+10-4
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ use crate::ast::{Lit, LitKind};
66
use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
77
use crate::ast::{Path, PathSegment};
88
use crate::token::{self, CommentKind, Token};
9-
use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree, TreeAndSpacing};
9+
use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
10+
use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing};
11+
use crate::tokenstream::{LazyTokenStream, TokenStream};
1012

1113
use rustc_index::bit_set::GrowableBitSet;
1214
use rustc_span::source_map::BytePos;
@@ -268,14 +270,18 @@ impl Attribute {
268270
}
269271
}
270272

271-
pub fn tokens(&self) -> TokenStream {
273+
pub fn tokens(&self) -> AttrAnnotatedTokenStream {
272274
match self.kind {
273275
AttrKind::Normal(_, ref tokens) => tokens
274276
.as_ref()
275277
.unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
276278
.create_token_stream(),
277-
AttrKind::DocComment(comment_kind, data) => TokenStream::from(TokenTree::Token(
278-
Token::new(token::DocComment(comment_kind, self.style, data), self.span),
279+
AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from((
280+
AttrAnnotatedTokenTree::Token(Token::new(
281+
token::DocComment(comment_kind, self.style, data),
282+
self.span,
283+
)),
284+
Spacing::Alone,
279285
)),
280286
}
281287
}

‎compiler/rustc_ast/src/mut_visit.rs

+45-4
Original file line numberDiff line numberDiff line change
@@ -630,6 +630,33 @@ pub fn noop_flat_map_param<T: MutVisitor>(mut param: Param, vis: &mut T) -> Smal
630630
smallvec![param]
631631
}
632632

633+
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
634+
pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) {
635+
match tt {
636+
AttrAnnotatedTokenTree::Token(token) => {
637+
visit_token(token, vis);
638+
}
639+
AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
640+
vis.visit_span(open);
641+
vis.visit_span(close);
642+
visit_attr_annotated_tts(tts, vis);
643+
}
644+
AttrAnnotatedTokenTree::Attributes(data) => {
645+
for attr in &mut *data.attrs {
646+
match &mut attr.kind {
647+
AttrKind::Normal(_, attr_tokens) => {
648+
visit_lazy_tts(attr_tokens, vis);
649+
}
650+
AttrKind::DocComment(..) => {
651+
vis.visit_span(&mut attr.span);
652+
}
653+
}
654+
}
655+
visit_lazy_tts_opt_mut(Some(&mut data.tokens), vis);
656+
}
657+
}
658+
}
659+
633660
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
634661
pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
635662
match tt {
@@ -652,16 +679,30 @@ pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T)
652679
}
653680
}
654681

655-
pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
682+
pub fn visit_attr_annotated_tts<T: MutVisitor>(
683+
AttrAnnotatedTokenStream(tts): &mut AttrAnnotatedTokenStream,
684+
vis: &mut T,
685+
) {
686+
if vis.token_visiting_enabled() && !tts.is_empty() {
687+
let tts = Lrc::make_mut(tts);
688+
visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis));
689+
}
690+
}
691+
692+
pub fn visit_lazy_tts_opt_mut<T: MutVisitor>(lazy_tts: Option<&mut LazyTokenStream>, vis: &mut T) {
656693
if vis.token_visiting_enabled() {
657-
visit_opt(lazy_tts, |lazy_tts| {
694+
if let Some(lazy_tts) = lazy_tts {
658695
let mut tts = lazy_tts.create_token_stream();
659-
visit_tts(&mut tts, vis);
696+
visit_attr_annotated_tts(&mut tts, vis);
660697
*lazy_tts = LazyTokenStream::new(tts);
661-
})
698+
}
662699
}
663700
}
664701

702+
pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
703+
visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis);
704+
}
705+
665706
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
666707
// Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
667708
// In practice the ident part is not actually used by specific visitors right now,

0 commit comments

Comments
 (0)