Skip to content

Commit 58624fc

Browse files
fee1-deadfee1-dead
committed
Change token::Ident to use IdentKind 
1 parent 8d90b43 commit 58624fc

File tree

25 files changed

+248
-141
lines changed

25 files changed

+248
-141
lines changed

Diff for: ‎compiler/rustc_ast/src/token.rs

+54-27
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,9 @@ impl Lit {
107107
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
108108
pub fn from_token(token: &Token) -> Option<Lit> {
109109
match token.uninterpolate().kind {
110-
Ident(name, false) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
110+
Ident(name, IdentKind::Default) if name.is_bool_lit() => {
111+
Some(Lit::new(Bool, name, None))
112+
}
111113
Literal(token_lit) => Some(token_lit),
112114
Interpolated(ref nt)
113115
if let NtExpr(expr) | NtLiteral(expr) = &nt.0
@@ -183,8 +185,8 @@ impl LitKind {
183185
}
184186
}
185187

186-
pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
187-
let ident_token = Token::new(Ident(name, is_raw), span);
188+
pub fn ident_can_begin_expr(name: Symbol, span: Span, kind: IdentKind) -> bool {
189+
let ident_token = Token::new(Ident(name, kind), span);
188190

189191
!ident_token.is_reserved_ident()
190192
|| ident_token.is_path_segment_keyword()
@@ -212,15 +214,37 @@ pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
212214
kw::Static,
213215
]
214216
.contains(&name)
217+
|| kind == IdentKind::Keyword
215218
}
216219

217-
fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool {
218-
let ident_token = Token::new(Ident(name, is_raw), span);
220+
fn ident_can_begin_type(name: Symbol, span: Span, kind: IdentKind) -> bool {
221+
let ident_token = Token::new(Ident(name, kind), span);
219222

220223
!ident_token.is_reserved_ident()
221224
|| ident_token.is_path_segment_keyword()
222225
|| [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
223226
.contains(&name)
227+
|| kind == IdentKind::Keyword
228+
}
229+
230+
#[derive(Clone, Copy, PartialEq, Eq, Debug, HashStable_Generic, Encodable, Decodable)]
231+
pub enum IdentKind {
232+
/// The usual identifiers (or, depending on the context, keywords): `v`, `union`, `await`, `loop`.
233+
Default,
234+
/// Raw identifiers: `r#just_an_ident`, `r#loop`.
235+
Raw,
236+
/// Forced keywords: `k#break`, `k#await`, `k#some_new_experimental_keyword`.
237+
Keyword,
238+
}
239+
240+
impl IdentKind {
241+
pub fn prefix(self) -> Option<&'static str> {
242+
match self {
243+
IdentKind::Default => None,
244+
IdentKind::Raw => Some("r#"),
245+
IdentKind::Keyword => Some("k#"),
246+
}
247+
}
224248
}
225249

226250
// SAFETY: due to the `Clone` impl below, all fields of all variants other than
@@ -298,10 +322,7 @@ pub enum TokenKind {
298322
/// Do not forget about `NtIdent` when you want to match on identifiers.
299323
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
300324
/// treat regular and interpolated identifiers in the same way.
301-
Ident(Symbol, /* is_raw */ bool),
302-
303-
/// A `k#ident` keyword
304-
Keyword(Symbol),
325+
Ident(Symbol, IdentKind),
305326

306327
/// Lifetime identifier token.
307328
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
@@ -415,7 +436,13 @@ impl Token {
415436

416437
/// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
417438
pub fn from_ast_ident(ident: Ident) -> Self {
418-
Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
439+
Token::new(
440+
Ident(
441+
ident.name,
442+
if ident.is_raw_guess() { IdentKind::Raw } else { IdentKind::Default },
443+
),
444+
ident.span,
445+
)
419446
}
420447

421448
/// For interpolated tokens, returns a span of the fragment to which the interpolated
@@ -442,7 +469,7 @@ impl Token {
442469
| ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => true,
443470

444471
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
445-
| Keyword(..) | Lifetime(..) | Interpolated(..) | Eof => false,
472+
| Lifetime(..) | Interpolated(..) | Eof => false,
446473
}
447474
}
448475

@@ -580,7 +607,7 @@ impl Token {
580607
pub fn can_begin_literal_maybe_minus(&self) -> bool {
581608
match self.uninterpolate().kind {
582609
Literal(..) | BinOp(Minus) => true,
583-
Ident(name, false) if name.is_bool_lit() => true,
610+
Ident(name, IdentKind::Default) if name.is_bool_lit() => true,
584611
Interpolated(ref nt) => match &nt.0 {
585612
NtLiteral(_) => true,
586613
NtExpr(e) => match &e.kind {
@@ -615,10 +642,10 @@ impl Token {
615642

616643
/// Returns an identifier if this token is an identifier.
617644
#[inline]
618-
pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
645+
pub fn ident(&self) -> Option<(Ident, IdentKind)> {
619646
// We avoid using `Token::uninterpolate` here because it's slow.
620647
match &self.kind {
621-
&Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
648+
&Ident(name, kind) => Some((Ident::new(name, self.span), kind)),
622649
Interpolated(nt) => match &nt.0 {
623650
NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
624651
_ => None,
@@ -711,46 +738,46 @@ impl Token {
711738

712739
/// Returns `true` if the token is a given keyword, `kw`.
713740
pub fn is_keyword(&self, kw: Symbol) -> bool {
714-
self.is_non_raw_ident_where(|id| id.name == kw)
741+
self.is_keywordable_ident_where(|id| id.name == kw)
715742
}
716743

717744
/// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this token is an identifier equal to `kw` ignoring the case.
718745
pub fn is_keyword_case(&self, kw: Symbol, case: Case) -> bool {
719746
self.is_keyword(kw)
720747
|| (case == Case::Insensitive
721-
&& self.is_non_raw_ident_where(|id| {
748+
&& self.is_keywordable_ident_where(|id| {
722749
id.name.as_str().to_lowercase() == kw.as_str().to_lowercase()
723750
}))
724751
}
725752

726753
pub fn is_path_segment_keyword(&self) -> bool {
727-
self.is_non_raw_ident_where(Ident::is_path_segment_keyword)
754+
self.is_keywordable_ident_where(Ident::is_path_segment_keyword)
728755
}
729756

730757
/// Returns true for reserved identifiers used internally for elided lifetimes,
731758
/// unnamed method parameters, crate root module, error recovery etc.
732759
pub fn is_special_ident(&self) -> bool {
733-
self.is_non_raw_ident_where(Ident::is_special)
760+
self.is_keywordable_ident_where(Ident::is_special)
734761
}
735762

736763
/// Returns `true` if the token is a keyword used in the language.
737764
pub fn is_used_keyword(&self) -> bool {
738-
self.is_non_raw_ident_where(Ident::is_used_keyword)
765+
self.is_keywordable_ident_where(Ident::is_used_keyword)
739766
}
740767

741768
/// Returns `true` if the token is a keyword reserved for possible future use.
742769
pub fn is_unused_keyword(&self) -> bool {
743-
self.is_non_raw_ident_where(Ident::is_unused_keyword)
770+
self.is_keywordable_ident_where(Ident::is_unused_keyword)
744771
}
745772

746773
/// Returns `true` if the token is either a special identifier or a keyword.
747774
pub fn is_reserved_ident(&self) -> bool {
748-
self.is_non_raw_ident_where(Ident::is_reserved)
775+
self.is_keywordable_ident_where(Ident::is_reserved)
749776
}
750777

751778
/// Returns `true` if the token is the identifier `true` or `false`.
752779
pub fn is_bool_lit(&self) -> bool {
753-
self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
780+
self.is_keywordable_ident_where(|id| id.name.is_bool_lit())
754781
}
755782

756783
pub fn is_numeric_lit(&self) -> bool {
@@ -766,9 +793,9 @@ impl Token {
766793
}
767794

768795
/// Returns `true` if the token is a non-raw identifier for which `pred` holds.
769-
pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
796+
pub fn is_keywordable_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
770797
match self.ident() {
771-
Some((id, false)) => pred(id),
798+
Some((id, IdentKind::Default | IdentKind::Keyword)) => pred(id),
772799
_ => false,
773800
}
774801
}
@@ -819,13 +846,13 @@ impl Token {
819846
_ => return None,
820847
},
821848
SingleQuote => match joint.kind {
822-
Ident(name, false) => Lifetime(Symbol::intern(&format!("'{name}"))),
849+
Ident(name, IdentKind::Default) => Lifetime(Symbol::intern(&format!("'{name}"))),
823850
_ => return None,
824851
},
825852

826853
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
827854
| DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
828-
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Keyword(..)
855+
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
829856
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
830857
};
831858

@@ -849,7 +876,7 @@ pub enum Nonterminal {
849876
NtPat(P<ast::Pat>),
850877
NtExpr(P<ast::Expr>),
851878
NtTy(P<ast::Ty>),
852-
NtIdent(Ident, /* is_raw */ bool),
879+
NtIdent(Ident, IdentKind),
853880
NtLifetime(Ident),
854881
NtLiteral(P<ast::Expr>),
855882
/// Stuff inside brackets for attributes

Diff for: ‎compiler/rustc_ast/src/tokenstream.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
1616
use crate::ast::{AttrStyle, StmtKind};
1717
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
18-
use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
18+
use crate::token::{self, Delimiter, IdentKind, Nonterminal, Token, TokenKind};
1919
use crate::AttrVec;
2020

2121
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -677,7 +677,7 @@ impl TokenStream {
677677
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
678678
Delimiter::Bracket,
679679
[
680-
TokenTree::token_alone(token::Ident(sym::doc, false), span),
680+
TokenTree::token_alone(token::Ident(sym::doc, IdentKind::Default), span),
681681
TokenTree::token_alone(token::Eq, span),
682682
TokenTree::token_alone(
683683
TokenKind::lit(token::StrRaw(num_of_hashes), data, None),

Diff for: ‎compiler/rustc_ast_pretty/src/pprust/state.rs

+6-5
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,9 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
268268
fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool);
269269

270270
fn print_ident(&mut self, ident: Ident) {
271-
self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string());
271+
self.word(
272+
IdentPrinter::for_ast_ident(ident, ident.is_raw_guess().then(|| "r#")).to_string(),
273+
);
272274
self.ann_post(ident)
273275
}
274276

@@ -715,7 +717,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
715717
token::NtBlock(e) => self.block_to_string(e),
716718
token::NtStmt(e) => self.stmt_to_string(e),
717719
token::NtPat(e) => self.pat_to_string(e),
718-
token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(*e, *is_raw).to_string(),
720+
&token::NtIdent(e, kind) => IdentPrinter::for_ast_ident(e, kind.prefix()).to_string(),
719721
token::NtLifetime(e) => e.to_string(),
720722
token::NtLiteral(e) => self.expr_to_string(e),
721723
token::NtVis(e) => self.vis_to_string(e),
@@ -778,10 +780,9 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
778780
token::Literal(lit) => literal_to_string(lit).into(),
779781

780782
/* Name components */
781-
token::Ident(s, is_raw) => {
782-
IdentPrinter::new(s, is_raw, convert_dollar_crate).to_string().into()
783+
token::Ident(s, kind) => {
784+
IdentPrinter::new(s, kind.prefix(), convert_dollar_crate).to_string().into()
783785
}
784-
token::Keyword(s) => format!("k#{s}").into(),
785786
token::Lifetime(s) => s.to_string().into(),
786787

787788
/* Other */

Diff for: ‎compiler/rustc_builtin_macros/src/asm.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use rustc_ast as ast;
22
use rustc_ast::ptr::P;
3-
use rustc_ast::token::{self, Delimiter};
3+
use rustc_ast::token::{self, Delimiter, IdentKind};
44
use rustc_ast::tokenstream::TokenStream;
55
use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
66
use rustc_errors::PResult;
@@ -416,7 +416,7 @@ fn parse_reg<'a>(
416416
) -> PResult<'a, ast::InlineAsmRegOrRegClass> {
417417
p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
418418
let result = match p.token.uninterpolate().kind {
419-
token::Ident(name, false) => ast::InlineAsmRegOrRegClass::RegClass(name),
419+
token::Ident(name, IdentKind::Default) => ast::InlineAsmRegOrRegClass::RegClass(name),
420420
token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
421421
*explicit_reg = true;
422422
ast::InlineAsmRegOrRegClass::Reg(symbol)

Diff for: ‎compiler/rustc_builtin_macros/src/assert/context.rs

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use rustc_ast::{
22
ptr::P,
33
token,
4-
token::Delimiter,
4+
token::{Delimiter, IdentKind},
55
tokenstream::{DelimSpan, TokenStream, TokenTree},
66
BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MethodCall, Mutability,
77
Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID,
@@ -170,7 +170,10 @@ impl<'cx, 'a> Context<'cx, 'a> {
170170
];
171171
let captures = self.capture_decls.iter().flat_map(|cap| {
172172
[
173-
TokenTree::token_joint_hidden(token::Ident(cap.ident.name, false), cap.ident.span),
173+
TokenTree::token_joint_hidden(
174+
token::Ident(cap.ident.name, IdentKind::Default),
175+
cap.ident.span,
176+
),
174177
TokenTree::token_alone(token::Comma, self.span),
175178
]
176179
});

Diff for: ‎compiler/rustc_expand/src/mbe/macro_check.rs

+5-2
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@
107107
use crate::errors;
108108
use crate::mbe::{KleeneToken, TokenTree};
109109

110-
use rustc_ast::token::{Delimiter, Token, TokenKind};
110+
use rustc_ast::token::{Delimiter, IdentKind, Token, TokenKind};
111111
use rustc_ast::{NodeId, DUMMY_NODE_ID};
112112
use rustc_data_structures::fx::FxHashMap;
113113
use rustc_errors::{DiagnosticMessage, MultiSpan};
@@ -409,7 +409,10 @@ fn check_nested_occurrences(
409409
match (state, tt) {
410410
(
411411
NestedMacroState::Empty,
412-
&TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }),
412+
&TokenTree::Token(Token {
413+
kind: TokenKind::Ident(name, IdentKind::Default | IdentKind::Keyword),
414+
..
415+
}),
413416
) => {
414417
if name == kw::MacroRules {
415418
state = NestedMacroState::MacroRules;

Diff for: ‎compiler/rustc_expand/src/mbe/macro_rules.rs

+18-5
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree, MatcherLoc};
99
use crate::mbe::transcribe::transcribe;
1010

1111
use rustc_ast as ast;
12-
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
12+
use rustc_ast::token::TokenKind::*;
13+
use rustc_ast::token::{self, Delimiter, IdentKind, NonterminalKind, Token, TokenKind};
1314
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
1415
use rustc_ast::{NodeId, DUMMY_NODE_ID};
1516
use rustc_ast_pretty::pprust;
@@ -1323,7 +1324,11 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13231324
match tok {
13241325
TokenTree::Token(token) => match token.kind {
13251326
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
1326-
Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
1327+
Ident(name, IdentKind::Default | IdentKind::Keyword)
1328+
if name == kw::If || name == kw::In =>
1329+
{
1330+
IsInFollow::Yes
1331+
}
13271332
_ => IsInFollow::No(TOKENS),
13281333
},
13291334
_ => IsInFollow::No(TOKENS),
@@ -1334,7 +1339,11 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13341339
match tok {
13351340
TokenTree::Token(token) => match token.kind {
13361341
FatArrow | Comma | Eq => IsInFollow::Yes,
1337-
Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
1342+
Ident(name, IdentKind::Default | IdentKind::Keyword)
1343+
if name == kw::If || name == kw::In =>
1344+
{
1345+
IsInFollow::Yes
1346+
}
13381347
_ => IsInFollow::No(TOKENS),
13391348
},
13401349
_ => IsInFollow::No(TOKENS),
@@ -1357,7 +1366,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13571366
| BinOp(token::Shr)
13581367
| Semi
13591368
| BinOp(token::Or) => IsInFollow::Yes,
1360-
Ident(name, false) if name == kw::As || name == kw::Where => {
1369+
Ident(name, IdentKind::Default | IdentKind::Keyword)
1370+
if name == kw::As || name == kw::Where =>
1371+
{
13611372
IsInFollow::Yes
13621373
}
13631374
_ => IsInFollow::No(TOKENS),
@@ -1385,7 +1396,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13851396
match tok {
13861397
TokenTree::Token(token) => match token.kind {
13871398
Comma => IsInFollow::Yes,
1388-
Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes,
1399+
Ident(name, kind) if kind == IdentKind::Raw || name != kw::Priv => {
1400+
IsInFollow::Yes
1401+
}
13891402
_ => {
13901403
if token.can_begin_type() {
13911404
IsInFollow::Yes

Diff for: ‎compiler/rustc_expand/src/mbe/metavar_expr.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use rustc_ast::token::{self, Delimiter};
1+
use rustc_ast::token::{self, Delimiter, IdentKind};
22
use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
33
use rustc_ast::{LitIntType, LitKind};
44
use rustc_ast_pretty::pprust;
@@ -142,7 +142,7 @@ fn parse_ident<'sess>(
142142
if let Some(tt) = iter.next()
143143
&& let TokenTree::Token(token, _) = tt
144144
{
145-
if let Some((elem, false)) = token.ident() {
145+
if let Some((elem, IdentKind::Default)) = token.ident() {
146146
return Ok(elem);
147147
}
148148
let token_str = pprust::token_to_string(token);

0 commit comments

Comments
 (0)