Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit beced67

Browse files
nnethercotennethercote
committed
Remove NtIdent and NtLifetime.
The extra span is now recorded in the new `TokenKind::InterpolatedIdent` and `TokenKind::InterpolatedLifetime`. These both consist of a single token, and so there's no operator precedence problems with inserting them directly into the token stream. The other way to do this would be to wrap the ident/lifetime in invisible delimiters, but there's a lot of code that assumes an interpolated ident/lifetime fits in a single token, and changing all that code to work with invisible delimiters would have been a pain. (Maybe it could be done in a follow-up.) This change might not seem like much of a win, but it's a first step toward the much bigger and long-desired removal of `Nonterminal` and `TokenKind::Interpolated`. That change is big and complex enough that it's worth doing this piece separately. (Indeed, this commit is based on part of a late commit in rust-lang#114647, a prior attempt at that big and complex change.)
1 parent 16ae355 commit beced67

File tree

11 files changed

+171
-112
lines changed

11 files changed

+171
-112
lines changed

‎compiler/rustc_ast/src/ast_traits.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,6 @@ impl HasTokens for Nonterminal {
240240
Nonterminal::NtPath(path) => path.tokens(),
241241
Nonterminal::NtVis(vis) => vis.tokens(),
242242
Nonterminal::NtBlock(block) => block.tokens(),
243-
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
244243
}
245244
}
246245
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
@@ -254,7 +253,6 @@ impl HasTokens for Nonterminal {
254253
Nonterminal::NtPath(path) => path.tokens_mut(),
255254
Nonterminal::NtVis(vis) => vis.tokens_mut(),
256255
Nonterminal::NtBlock(block) => block.tokens_mut(),
257-
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
258256
}
259257
}
260258
}

‎compiler/rustc_ast/src/mut_visit.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -783,6 +783,9 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
783783
*span = ident.span;
784784
return; // Avoid visiting the span for the second time.
785785
}
786+
token::InterpolatedIdent(ident, _) | token::InterpolatedLifetime(ident) => {
787+
vis.visit_ident(ident);
788+
}
786789
token::Interpolated(nt) => {
787790
let nt = Lrc::make_mut(nt);
788791
visit_nonterminal(nt, vis);
@@ -834,8 +837,6 @@ pub fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T
834837
token::NtPat(pat) => vis.visit_pat(pat),
835838
token::NtExpr(expr) => vis.visit_expr(expr),
836839
token::NtTy(ty) => vis.visit_ty(ty),
837-
token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
838-
token::NtLifetime(ident) => vis.visit_ident(ident),
839840
token::NtLiteral(expr) => vis.visit_expr(expr),
840841
token::NtMeta(item) => {
841842
let AttrItem { path, args, tokens } = item.deref_mut();

‎compiler/rustc_ast/src/token.rs‎

Lines changed: 71 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -314,15 +314,24 @@ pub enum TokenKind {
314314
Literal(Lit),
315315

316316
/// Identifier token.
317-
/// Do not forget about `NtIdent` when you want to match on identifiers.
317+
/// Do not forget about `InterpolatedIdent` when you want to match on identifiers.
318318
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
319319
/// treat regular and interpolated identifiers in the same way.
320320
Ident(Symbol, IdentIsRaw),
321+
/// This identifier (and its span) is the identifier passed to the
322+
/// declarative macro. The span in the surrounding `Token` is the span of
323+
/// the `ident` metavariable in the macro's RHS.
324+
InterpolatedIdent(Ident, IdentIsRaw),
325+
321326
/// Lifetime identifier token.
322-
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
327+
/// Do not forget about `InterpolatedLIfetime` when you want to match on lifetime identifiers.
323328
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
324329
/// treat regular and interpolated lifetime identifiers in the same way.
325330
Lifetime(Symbol),
331+
/// This identifier (and its span) is the lifetime passed to the
332+
/// declarative macro. The span in the surrounding `Token` is the span of
333+
/// the `lifetime` metavariable in the macro's RHS.
334+
InterpolatedLifetime(Ident),
326335

327336
/// An embedded AST node, as produced by a macro. This only exists for
328337
/// historical reasons. We'd like to get rid of it, for multiple reasons.
@@ -444,8 +453,9 @@ impl Token {
444453
/// Note that keywords are also identifiers, so they should use this
445454
/// if they keep spans or perform edition checks.
446455
pub fn uninterpolated_span(&self) -> Span {
447-
match &self.kind {
448-
Interpolated(nt) => nt.use_span(),
456+
match self.kind {
457+
InterpolatedIdent(ident, _) | InterpolatedLifetime(ident) => ident.span,
458+
Interpolated(ref nt) => nt.use_span(),
449459
_ => self.span,
450460
}
451461
}
@@ -462,8 +472,16 @@ impl Token {
462472
true
463473
}
464474

465-
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
466-
| Lifetime(..) | Interpolated(..) | Eof => false,
475+
OpenDelim(..)
476+
| CloseDelim(..)
477+
| Literal(..)
478+
| DocComment(..)
479+
| Ident(..)
480+
| InterpolatedIdent(..)
481+
| Lifetime(..)
482+
| InterpolatedLifetime(..)
483+
| Interpolated(..)
484+
| Eof => false,
467485
}
468486
}
469487

@@ -613,14 +631,11 @@ impl Token {
613631
/// into the regular identifier or lifetime token it refers to,
614632
/// otherwise returns the original token.
615633
pub fn uninterpolate(&self) -> Cow<'_, Token> {
616-
match &self.kind {
617-
Interpolated(nt) => match &**nt {
618-
NtIdent(ident, is_raw) => {
619-
Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
620-
}
621-
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
622-
_ => Cow::Borrowed(self),
623-
},
634+
match self.kind {
635+
InterpolatedIdent(ident, is_raw) => {
636+
Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
637+
}
638+
InterpolatedLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
624639
_ => Cow::Borrowed(self),
625640
}
626641
}
@@ -629,12 +644,9 @@ impl Token {
629644
#[inline]
630645
pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> {
631646
// We avoid using `Token::uninterpolate` here because it's slow.
632-
match &self.kind {
633-
&Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
634-
Interpolated(nt) => match &**nt {
635-
NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
636-
_ => None,
637-
},
647+
match self.kind {
648+
Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
649+
InterpolatedIdent(ident, is_raw) => Some((ident, is_raw)),
638650
_ => None,
639651
}
640652
}
@@ -643,12 +655,9 @@ impl Token {
643655
#[inline]
644656
pub fn lifetime(&self) -> Option<Ident> {
645657
// We avoid using `Token::uninterpolate` here because it's slow.
646-
match &self.kind {
647-
&Lifetime(name) => Some(Ident::new(name, self.span)),
648-
Interpolated(nt) => match &**nt {
649-
NtLifetime(ident) => Some(*ident),
650-
_ => None,
651-
},
658+
match self.kind {
659+
Lifetime(name) => Some(Ident::new(name, self.span)),
660+
InterpolatedLifetime(ident) => Some(ident),
652661
_ => None,
653662
}
654663
}
@@ -835,10 +844,36 @@ impl Token {
835844
_ => return None,
836845
},
837846

838-
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
839-
| DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
840-
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
841-
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
847+
Le
848+
| EqEq
849+
| Ne
850+
| Ge
851+
| AndAnd
852+
| OrOr
853+
| Tilde
854+
| BinOpEq(..)
855+
| At
856+
| DotDotDot
857+
| DotDotEq
858+
| Comma
859+
| Semi
860+
| PathSep
861+
| RArrow
862+
| LArrow
863+
| FatArrow
864+
| Pound
865+
| Dollar
866+
| Question
867+
| OpenDelim(..)
868+
| CloseDelim(..)
869+
| Literal(..)
870+
| Ident(..)
871+
| InterpolatedIdent(..)
872+
| Lifetime(..)
873+
| InterpolatedLifetime(..)
874+
| Interpolated(..)
875+
| DocComment(..)
876+
| Eof => return None,
842877
};
843878

844879
Some(Token::new(kind, self.span.to(joint.span)))
@@ -861,9 +896,6 @@ pub enum Nonterminal {
861896
NtPat(P<ast::Pat>),
862897
NtExpr(P<ast::Expr>),
863898
NtTy(P<ast::Ty>),
864-
/// The span is for the identifier argument passed to the macro.
865-
NtIdent(Ident, IdentIsRaw),
866-
NtLifetime(Ident),
867899
NtLiteral(P<ast::Expr>),
868900
/// Stuff inside brackets for attributes
869901
NtMeta(P<ast::AttrItem>),
@@ -958,7 +990,6 @@ impl Nonterminal {
958990
NtPat(pat) => pat.span,
959991
NtExpr(expr) | NtLiteral(expr) => expr.span,
960992
NtTy(ty) => ty.span,
961-
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
962993
NtMeta(attr_item) => attr_item.span(),
963994
NtPath(path) => path.span,
964995
NtVis(vis) => vis.span,
@@ -974,8 +1005,6 @@ impl Nonterminal {
9741005
NtExpr(..) => "expression",
9751006
NtLiteral(..) => "literal",
9761007
NtTy(..) => "type",
977-
NtIdent(..) => "identifier",
978-
NtLifetime(..) => "lifetime",
9791008
NtMeta(..) => "attribute",
9801009
NtPath(..) => "path",
9811010
NtVis(..) => "visibility",
@@ -984,18 +1013,12 @@ impl Nonterminal {
9841013
}
9851014

9861015
impl PartialEq for Nonterminal {
987-
fn eq(&self, rhs: &Self) -> bool {
988-
match (self, rhs) {
989-
(NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
990-
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
991-
}
992-
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
993-
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
994-
// correctly based on data from AST. This will prevent them from matching each other
995-
// in macros. The comparison will become possible only when each nonterminal has an
996-
// attached token stream from which it was parsed.
997-
_ => false,
998-
}
1016+
fn eq(&self, _rhs: &Self) -> bool {
1017+
// FIXME: Assume that all nonterminals are not equal, we can't compare them
1018+
// correctly based on data from AST. This will prevent them from matching each other
1019+
// in macros. The comparison will become possible only when each nonterminal has an
1020+
// attached token stream from which it was parsed.
1021+
false
9991022
}
10001023
}
10011024

@@ -1008,12 +1031,10 @@ impl fmt::Debug for Nonterminal {
10081031
NtPat(..) => f.pad("NtPat(..)"),
10091032
NtExpr(..) => f.pad("NtExpr(..)"),
10101033
NtTy(..) => f.pad("NtTy(..)"),
1011-
NtIdent(..) => f.pad("NtIdent(..)"),
10121034
NtLiteral(..) => f.pad("NtLiteral(..)"),
10131035
NtMeta(..) => f.pad("NtMeta(..)"),
10141036
NtPath(..) => f.pad("NtPath(..)"),
10151037
NtVis(..) => f.pad("NtVis(..)"),
1016-
NtLifetime(..) => f.pad("NtLifetime(..)"),
10171038
}
10181039
}
10191040
}

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -467,12 +467,6 @@ impl TokenStream {
467467

468468
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
469469
match nt {
470-
Nonterminal::NtIdent(ident, is_raw) => {
471-
TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span)
472-
}
473-
Nonterminal::NtLifetime(ident) => {
474-
TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
475-
}
476470
Nonterminal::NtItem(item) => TokenStream::from_ast(item),
477471
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
478472
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
@@ -490,11 +484,17 @@ impl TokenStream {
490484
}
491485

492486
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
493-
match &token.kind {
494-
token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => {
495-
TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing)
487+
match token.kind {
488+
token::InterpolatedIdent(ident, is_raw) => {
489+
TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
496490
}
497-
token::Interpolated(nt) => TokenTree::Delimited(
491+
token::InterpolatedLifetime(ident) => TokenTree::Delimited(
492+
DelimSpan::from_single(token.span),
493+
DelimSpacing::new(Spacing::JointHidden, spacing),
494+
Delimiter::Invisible,
495+
TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
496+
),
497+
token::Interpolated(ref nt) => TokenTree::Delimited(
498498
DelimSpan::from_single(token.span),
499499
DelimSpacing::new(Spacing::JointHidden, spacing),
500500
Delimiter::Invisible,
@@ -517,7 +517,12 @@ impl TokenStream {
517517
pub fn flattened(&self) -> TokenStream {
518518
fn can_skip(stream: &TokenStream) -> bool {
519519
stream.trees().all(|tree| match tree {
520-
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
520+
TokenTree::Token(token, _) => !matches!(
521+
token.kind,
522+
token::InterpolatedIdent(..)
523+
| token::InterpolatedLifetime(..)
524+
| token::Interpolated(..)
525+
),
521526
TokenTree::Delimited(.., inner) => can_skip(inner),
522527
})
523528
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -852,8 +852,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
852852
token::NtBlock(e) => self.block_to_string(e),
853853
token::NtStmt(e) => self.stmt_to_string(e),
854854
token::NtPat(e) => self.pat_to_string(e),
855-
&token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(),
856-
token::NtLifetime(e) => e.to_string(),
857855
token::NtLiteral(e) => self.expr_to_string(e),
858856
token::NtVis(e) => self.vis_to_string(e),
859857
}
@@ -915,10 +913,16 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
915913
token::Literal(lit) => literal_to_string(lit).into(),
916914

917915
/* Name components */
918-
token::Ident(s, is_raw) => {
919-
IdentPrinter::new(s, is_raw.into(), convert_dollar_crate).to_string().into()
916+
token::Ident(name, is_raw) => {
917+
IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into()
920918
}
921-
token::Lifetime(s) => s.to_string().into(),
919+
token::InterpolatedIdent(ident, is_raw) => {
920+
IdentPrinter::new(ident.name, is_raw.into(), convert_dollar_crate)
921+
.to_string()
922+
.into()
923+
}
924+
token::Lifetime(name) => name.to_string().into(),
925+
token::InterpolatedLifetime(ident) => ident.name.to_string().into(),
922926

923927
/* Other */
924928
token::DocComment(comment_kind, attr_style, data) => {

‎compiler/rustc_expand/src/config.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,17 @@ impl<'a> StripUnconfigured<'a> {
199199
inner = self.configure_tokens(&inner);
200200
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
201201
}
202-
AttrTokenTree::Token(ref token, _)
203-
if let TokenKind::Interpolated(nt) = &token.kind =>
204-
{
205-
panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt);
202+
AttrTokenTree::Token(
203+
Token {
204+
kind:
205+
TokenKind::InterpolatedIdent(..)
206+
| TokenKind::InterpolatedLifetime(..)
207+
| TokenKind::Interpolated(..),
208+
..
209+
},
210+
_,
211+
) => {
212+
panic!("Nonterminal should have been flattened: {:?}", tree);
206213
}
207214
AttrTokenTree::Token(token, spacing) => {
208215
Some(AttrTokenTree::Token(token, spacing)).into_iter()

‎compiler/rustc_expand/src/mbe/transcribe.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -256,10 +256,17 @@ pub(super) fn transcribe<'a>(
256256
// without wrapping them into groups.
257257
maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker)
258258
}
259+
MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => {
260+
marker.visit_span(&mut sp);
261+
let kind = token::InterpolatedIdent(*ident, *is_raw);
262+
TokenTree::token_alone(kind, sp)
263+
}
264+
MatchedSingle(ParseNtResult::Lifetime(ident)) => {
265+
marker.visit_span(&mut sp);
266+
let kind = token::InterpolatedLifetime(*ident);
267+
TokenTree::token_alone(kind, sp)
268+
}
259269
MatchedSingle(ParseNtResult::Nt(nt)) => {
260-
// Other variables are emitted into the output stream as groups with
261-
// `Delimiter::Invisible` to maintain parsing priorities.
262-
// `Interpolated` is currently used for such groups in rustc parser.
263270
marker.visit_span(&mut sp);
264271
TokenTree::token_alone(token::Interpolated(nt.clone()), sp)
265272
}

0 commit comments

Comments
 (0)