Skip to content

Commit 77b0ed7

Browse files
petrochenkovpetrochenkov
committed
proc_macro: Stop flattening groups with dummy spans
1 parent 50fc24d commit 77b0ed7

File tree

18 files changed

+76
-50
lines changed

18 files changed

+76
-50
lines changed

Diff for: ‎src/librustc_ast/attr/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,7 @@ impl MetaItem {
475475
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
476476
Path { span, segments }
477477
}
478-
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
478+
Some(TokenTree::Token(Token { kind: token::Interpolated(nt, _), .. })) => match *nt {
479479
token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
480480
token::Nonterminal::NtPath(ref path) => path.clone(),
481481
_ => return None,

Diff for: ‎src/librustc_ast/mut_visit.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -656,7 +656,7 @@ pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
656656
*span = ident.span;
657657
return; // Avoid visiting the span for the second time.
658658
}
659-
token::Interpolated(nt) => {
659+
token::Interpolated(nt, _) => {
660660
let mut nt = Lrc::make_mut(nt);
661661
vis.visit_interpolated(&mut nt);
662662
}

Diff for: ‎src/librustc_ast/token.rs

+20-11
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,15 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool {
182182
.contains(&name)
183183
}
184184

185+
/// A hack used to pass AST fragments to attribute and derive macros
186+
/// as a single nonterminal token instead of a token stream.
187+
/// FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
188+
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
189+
pub enum FlattenGroup {
190+
Yes,
191+
No,
192+
}
193+
185194
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
186195
pub enum TokenKind {
187196
/* Expression-operator symbols. */
@@ -236,7 +245,7 @@ pub enum TokenKind {
236245
/// treat regular and interpolated lifetime identifiers in the same way.
237246
Lifetime(Symbol),
238247

239-
Interpolated(Lrc<Nonterminal>),
248+
Interpolated(Lrc<Nonterminal>, FlattenGroup),
240249

241250
// Can be expanded into several tokens.
242251
/// A doc comment.
@@ -343,7 +352,7 @@ impl Token {
343352
/// if they keep spans or perform edition checks.
344353
pub fn uninterpolated_span(&self) -> Span {
345354
match &self.kind {
346-
Interpolated(nt) => nt.span(),
355+
Interpolated(nt, _) => nt.span(),
347356
_ => self.span,
348357
}
349358
}
@@ -382,7 +391,7 @@ impl Token {
382391
ModSep | // global path
383392
Lifetime(..) | // labeled loop
384393
Pound => true, // expression attributes
385-
Interpolated(ref nt) => match **nt {
394+
Interpolated(ref nt, _) => match **nt {
386395
NtLiteral(..) |
387396
NtExpr(..) |
388397
NtBlock(..) |
@@ -408,7 +417,7 @@ impl Token {
408417
Lifetime(..) | // lifetime bound in trait object
409418
Lt | BinOp(Shl) | // associated path
410419
ModSep => true, // global path
411-
Interpolated(ref nt) => match **nt {
420+
Interpolated(ref nt, _) => match **nt {
412421
NtTy(..) | NtPath(..) => true,
413422
_ => false,
414423
},
@@ -420,7 +429,7 @@ impl Token {
420429
pub fn can_begin_const_arg(&self) -> bool {
421430
match self.kind {
422431
OpenDelim(Brace) => true,
423-
Interpolated(ref nt) => match **nt {
432+
Interpolated(ref nt, _) => match **nt {
424433
NtExpr(..) | NtBlock(..) | NtLiteral(..) => true,
425434
_ => false,
426435
},
@@ -455,7 +464,7 @@ impl Token {
455464
match self.uninterpolate().kind {
456465
Literal(..) | BinOp(Minus) => true,
457466
Ident(name, false) if name.is_bool_lit() => true,
458-
Interpolated(ref nt) => match &**nt {
467+
Interpolated(ref nt, _) => match &**nt {
459468
NtLiteral(_) => true,
460469
NtExpr(e) => match &e.kind {
461470
ast::ExprKind::Lit(_) => true,
@@ -476,7 +485,7 @@ impl Token {
476485
// otherwise returns the original token.
477486
pub fn uninterpolate(&self) -> Cow<'_, Token> {
478487
match &self.kind {
479-
Interpolated(nt) => match **nt {
488+
Interpolated(nt, _) => match **nt {
480489
NtIdent(ident, is_raw) => {
481490
Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
482491
}
@@ -523,7 +532,7 @@ impl Token {
523532

524533
/// Returns `true` if the token is an interpolated path.
525534
fn is_path(&self) -> bool {
526-
if let Interpolated(ref nt) = self.kind {
535+
if let Interpolated(ref nt, _) = self.kind {
527536
if let NtPath(..) = **nt {
528537
return true;
529538
}
@@ -535,7 +544,7 @@ impl Token {
535544
/// That is, is this a pre-parsed expression dropped into the token stream
536545
/// (which happens while parsing the result of macro expansion)?
537546
pub fn is_whole_expr(&self) -> bool {
538-
if let Interpolated(ref nt) = self.kind {
547+
if let Interpolated(ref nt, _) = self.kind {
539548
if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt {
540549
return true;
541550
}
@@ -546,7 +555,7 @@ impl Token {
546555

547556
// Is the token an interpolated block (`$b:block`)?
548557
pub fn is_whole_block(&self) -> bool {
549-
if let Interpolated(ref nt) = self.kind {
558+
if let Interpolated(ref nt, _) = self.kind {
550559
if let NtBlock(..) = **nt {
551560
return true;
552561
}
@@ -724,7 +733,7 @@ impl Token {
724733
b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
725734
}
726735

727-
(&Interpolated(_), &Interpolated(_)) => false,
736+
(&Interpolated(..), &Interpolated(..)) => false,
728737

729738
_ => panic!("forgot to add a token?"),
730739
}

Diff for: ‎src/librustc_ast/util/literal.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ impl Lit {
205205
token::Lit::new(token::Bool, name, None)
206206
}
207207
token::Literal(lit) => lit,
208-
token::Interpolated(ref nt) => {
208+
token::Interpolated(ref nt, _) => {
209209
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
210210
if let ast::ExprKind::Lit(lit) = &expr.kind {
211211
return Ok(lit.clone());

Diff for: ‎src/librustc_ast_lowering/lib.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1027,7 +1027,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
10271027

10281028
fn lower_token(&mut self, token: Token) -> TokenStream {
10291029
match token.kind {
1030-
token::Interpolated(nt) => {
1030+
token::Interpolated(nt, _) => {
10311031
let tts = (self.nt_to_tokenstream)(&nt, &self.sess.parse_sess, token.span);
10321032
self.lower_token_stream(tts)
10331033
}

Diff for: ‎src/librustc_ast_pretty/pprust.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option<Span>)
266266
token::Shebang(s) => format!("/* shebang: {}*/", s),
267267
token::Unknown(s) => s.to_string(),
268268

269-
token::Interpolated(ref nt) => nonterminal_to_string(nt),
269+
token::Interpolated(ref nt, _) => nonterminal_to_string(nt),
270270
}
271271
}
272272

Diff for: ‎src/librustc_expand/base.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use crate::module::DirectoryOwnership;
44
use rustc_ast::ast::{self, Attribute, NodeId, PatKind};
55
use rustc_ast::mut_visit::{self, MutVisitor};
66
use rustc_ast::ptr::P;
7-
use rustc_ast::token;
7+
use rustc_ast::token::{self, FlattenGroup};
88
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
99
use rustc_ast::visit::{AssocCtxt, Visitor};
1010
use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
@@ -142,7 +142,7 @@ impl Annotatable {
142142
| Annotatable::StructField(..)
143143
| Annotatable::Variant(..) => panic!("unexpected annotatable"),
144144
};
145-
TokenTree::token(token::Interpolated(Lrc::new(nt)), DUMMY_SP).into()
145+
TokenTree::token(token::Interpolated(Lrc::new(nt), FlattenGroup::Yes), DUMMY_SP).into()
146146
}
147147

148148
pub fn expect_item(self) -> P<ast::Item> {
@@ -374,7 +374,7 @@ where
374374
impl MutVisitor for AvoidInterpolatedIdents {
375375
fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) {
376376
if let tokenstream::TokenTree::Token(token) = tt {
377-
if let token::Interpolated(nt) = &token.kind {
377+
if let token::Interpolated(nt, _) = &token.kind {
378378
if let token::NtIdent(ident, is_raw) = **nt {
379379
*tt = tokenstream::TokenTree::token(
380380
token::Ident(ident.name, is_raw),

Diff for: ‎src/librustc_expand/mbe/macro_parser.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -785,12 +785,12 @@ fn may_begin_with(token: &Token, name: Symbol) -> bool {
785785
sym::literal => token.can_begin_literal_maybe_minus(),
786786
sym::vis => match token.kind {
787787
// The follow-set of :vis + "priv" keyword + interpolated
788-
token::Comma | token::Ident(..) | token::Interpolated(_) => true,
788+
token::Comma | token::Ident(..) | token::Interpolated(..) => true,
789789
_ => token.can_begin_type(),
790790
},
791791
sym::block => match token.kind {
792792
token::OpenDelim(token::Brace) => true,
793-
token::Interpolated(ref nt) => match **nt {
793+
token::Interpolated(ref nt, _) => match **nt {
794794
token::NtItem(_)
795795
| token::NtPat(_)
796796
| token::NtTy(_)
@@ -804,7 +804,7 @@ fn may_begin_with(token: &Token, name: Symbol) -> bool {
804804
},
805805
sym::path | sym::meta => match token.kind {
806806
token::ModSep | token::Ident(..) => true,
807-
token::Interpolated(ref nt) => match **nt {
807+
token::Interpolated(ref nt, _) => match **nt {
808808
token::NtPath(_) | token::NtMeta(_) => true,
809809
_ => may_be_ident(&nt),
810810
},
@@ -823,12 +823,12 @@ fn may_begin_with(token: &Token, name: Symbol) -> bool {
823823
token::ModSep | // path
824824
token::Lt | // path (UFCS constant)
825825
token::BinOp(token::Shl) => true, // path (double UFCS)
826-
token::Interpolated(ref nt) => may_be_ident(nt),
826+
token::Interpolated(ref nt, _) => may_be_ident(nt),
827827
_ => false,
828828
},
829829
sym::lifetime => match token.kind {
830830
token::Lifetime(_) => true,
831-
token::Interpolated(ref nt) => match **nt {
831+
token::Interpolated(ref nt, _) => match **nt {
832832
token::NtLifetime(_) | token::NtTT(_) => true,
833833
_ => false,
834834
},

Diff for: ‎src/librustc_expand/mbe/transcribe.rs

+5-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
44

55
use rustc_ast::ast::MacCall;
66
use rustc_ast::mut_visit::{self, MutVisitor};
7-
use rustc_ast::token::{self, NtTT, Token};
7+
use rustc_ast::token::{self, FlattenGroup, NtTT, Token};
88
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
99
use rustc_data_structures::fx::FxHashMap;
1010
use rustc_data_structures::sync::Lrc;
@@ -240,7 +240,10 @@ pub(super) fn transcribe<'a>(
240240
result.push(tt.clone().into());
241241
} else {
242242
marker.visit_span(&mut sp);
243-
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
243+
let token = TokenTree::token(
244+
token::Interpolated(nt.clone(), FlattenGroup::No),
245+
sp,
246+
);
244247
result.push(token.into());
245248
}
246249
} else {

Diff for: ‎src/librustc_expand/proc_macro.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use crate::base::{self, *};
22
use crate::proc_macro_server;
33

44
use rustc_ast::ast::{self, ItemKind, MetaItemKind, NestedMetaItem};
5-
use rustc_ast::token;
5+
use rustc_ast::token::{self, FlattenGroup};
66
use rustc_ast::tokenstream::{self, TokenStream};
77
use rustc_data_structures::sync::Lrc;
88
use rustc_errors::{Applicability, ErrorReported};
@@ -102,7 +102,7 @@ impl MultiItemModifier for ProcMacroDerive {
102102
}
103103
}
104104

105-
let token = token::Interpolated(Lrc::new(token::NtItem(item)));
105+
let token = token::Interpolated(Lrc::new(token::NtItem(item)), FlattenGroup::Yes);
106106
let input = tokenstream::TokenTree::token(token, DUMMY_SP).into();
107107

108108
let server = proc_macro_server::Rustc::new(ecx);

Diff for: ‎src/librustc_expand/proc_macro_server.rs

+26-12
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::base::ExtCtxt;
22

33
use rustc_ast::ast;
4-
use rustc_ast::token;
4+
use rustc_ast::token::{self, FlattenGroup};
55
use rustc_ast::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
66
use rustc_ast::util::comments;
77
use rustc_ast_pretty::pprust;
@@ -60,7 +60,12 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
6060
let Token { kind, span } = match tree {
6161
tokenstream::TokenTree::Delimited(span, delim, tts) => {
6262
let delimiter = Delimiter::from_internal(delim);
63-
return TokenTree::Group(Group { delimiter, stream: tts, span });
63+
return TokenTree::Group(Group {
64+
delimiter,
65+
stream: tts,
66+
span,
67+
flatten: FlattenGroup::No,
68+
});
6469
}
6570
tokenstream::TokenTree::Token(token) => token,
6671
};
@@ -167,19 +172,21 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
167172
delimiter: Delimiter::Bracket,
168173
stream,
169174
span: DelimSpan::from_single(span),
175+
flatten: FlattenGroup::No,
170176
}));
171177
if style == ast::AttrStyle::Inner {
172178
stack.push(tt!(Punct::new('!', false)));
173179
}
174180
tt!(Punct::new('#', false))
175181
}
176182

177-
Interpolated(nt) => {
183+
Interpolated(nt, flatten) => {
178184
let stream = nt_to_tokenstream(&nt, sess, span);
179185
TokenTree::Group(Group {
180186
delimiter: Delimiter::None,
181187
stream,
182188
span: DelimSpan::from_single(span),
189+
flatten,
183190
})
184191
}
185192

@@ -195,7 +202,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
195202

196203
let (ch, joint, span) = match self {
197204
TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
198-
TokenTree::Group(Group { delimiter, stream, span }) => {
205+
TokenTree::Group(Group { delimiter, stream, span, .. }) => {
199206
return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream)
200207
.into();
201208
}
@@ -283,6 +290,10 @@ pub struct Group {
283290
delimiter: Delimiter,
284291
stream: TokenStream,
285292
span: DelimSpan,
293+
/// A hack used to pass AST fragments to attribute and derive macros
294+
/// as a single nonterminal token instead of a token stream.
295+
/// FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
296+
flatten: FlattenGroup,
286297
}
287298

288299
#[derive(Copy, Clone, PartialEq, Eq, Hash)]
@@ -437,14 +448,12 @@ impl server::TokenStreamIter for Rustc<'_> {
437448
let next = iter.cursor.next_with_joint()?;
438449
Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
439450
})?;
440-
// HACK: The condition "dummy span + group with empty delimiter" represents an AST
441-
// fragment approximately converted into a token stream. This may happen, for
442-
// example, with inputs to proc macro attributes, including derives. Such "groups"
443-
// need to flattened during iteration over stream's token trees.
444-
// Eventually this needs to be removed in favor of keeping original token trees
445-
// and not doing the roundtrip through AST.
451+
// A hack used to pass AST fragments to attribute and derive macros
452+
// as a single nonterminal token instead of a token stream.
453+
// Such token needs to be "unwrapped" and not represented as a delimited group.
454+
// FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
446455
if let TokenTree::Group(ref group) = tree {
447-
if group.delimiter == Delimiter::None && group.span.entire().is_dummy() {
456+
if matches!(group.flatten, FlattenGroup::Yes) {
448457
iter.cursor.append(group.stream.clone());
449458
continue;
450459
}
@@ -456,7 +465,12 @@ impl server::TokenStreamIter for Rustc<'_> {
456465

457466
impl server::Group for Rustc<'_> {
458467
fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
459-
Group { delimiter, stream, span: DelimSpan::from_single(server::Span::call_site(self)) }
468+
Group {
469+
delimiter,
470+
stream,
471+
span: DelimSpan::from_single(server::Span::call_site(self)),
472+
flatten: FlattenGroup::No,
473+
}
460474
}
461475
fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
462476
group.delimiter

Diff for: ‎src/librustc_parse/parser/attr.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ impl<'a> Parser<'a> {
155155
/// The delimiters or `=` are still put into the resulting token stream.
156156
pub fn parse_attr_item(&mut self) -> PResult<'a, ast::AttrItem> {
157157
let item = match self.token.kind {
158-
token::Interpolated(ref nt) => match **nt {
158+
token::Interpolated(ref nt, _) => match **nt {
159159
Nonterminal::NtMeta(ref item) => Some(item.clone().into_inner()),
160160
_ => None,
161161
},
@@ -254,7 +254,7 @@ impl<'a> Parser<'a> {
254254
/// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
255255
pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
256256
let nt_meta = match self.token.kind {
257-
token::Interpolated(ref nt) => match **nt {
257+
token::Interpolated(ref nt, _) => match **nt {
258258
token::NtMeta(ref e) => Some(e.clone()),
259259
_ => None,
260260
},

Diff for: ‎src/librustc_parse/parser/expr.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ use std::mem;
2626
/// `token::Interpolated` tokens.
2727
macro_rules! maybe_whole_expr {
2828
($p:expr) => {
29-
if let token::Interpolated(nt) = &$p.token.kind {
29+
if let token::Interpolated(nt, _) = &$p.token.kind {
3030
match &**nt {
3131
token::NtExpr(e) | token::NtLiteral(e) => {
3232
let e = e.clone();

0 commit comments

Comments
 (0)