Skip to content

Commit b2605c1

Browse files
petrochenkovpetrochenkov
committed
parser: token -> normalized_token, nonnormalized_token -> token 
1 parent 59261f0 commit b2605c1

File tree

14 files changed

+88
-89
lines changed

14 files changed

+88
-89
lines changed

Diff for: ‎src/librustc_builtin_macros/format.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ fn parse_args<'a>(
158158
} // accept trailing commas
159159
if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
160160
named = true;
161-
let name = if let token::Ident(name, _) = p.token.kind {
161+
let name = if let token::Ident(name, _) = p.normalized_token.kind {
162162
p.bump();
163163
name
164164
} else {

Diff for: ‎src/librustc_expand/mbe/macro_parser.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -889,9 +889,8 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
889889
// this could be handled like a token, since it is one
890890
sym::ident => {
891891
if let Some((name, is_raw)) = get_macro_name(&p.token) {
892-
let span = p.token.span;
893892
p.bump();
894-
token::NtIdent(Ident::new(name, span), is_raw)
893+
token::NtIdent(Ident::new(name, p.normalized_prev_token.span), is_raw)
895894
} else {
896895
let token_str = pprust::token_to_string(&p.token);
897896
let msg = &format!("expected ident, found {}", &token_str);

Diff for: ‎src/librustc_parse/parser/diagnostics.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -192,12 +192,12 @@ impl<'a> Parser<'a> {
192192
TokenKind::CloseDelim(token::DelimToken::Brace),
193193
TokenKind::CloseDelim(token::DelimToken::Paren),
194194
];
195-
if let token::Ident(name, false) = self.token.kind {
196-
if Ident::new(name, self.token.span).is_raw_guess()
195+
if let token::Ident(name, false) = self.normalized_token.kind {
196+
if Ident::new(name, self.normalized_token.span).is_raw_guess()
197197
&& self.look_ahead(1, |t| valid_follow.contains(&t.kind))
198198
{
199199
err.span_suggestion(
200-
self.token.span,
200+
self.normalized_token.span,
201201
"you can escape reserved keywords to use them as identifiers",
202202
format!("r#{}", name),
203203
Applicability::MaybeIncorrect,

Diff for: ‎src/librustc_parse/parser/expr.rs

+18-21
Original file line numberDiff line numberDiff line change
@@ -97,15 +97,14 @@ impl<'a> Parser<'a> {
9797
fn parse_expr_catch_underscore(&mut self) -> PResult<'a, P<Expr>> {
9898
match self.parse_expr() {
9999
Ok(expr) => Ok(expr),
100-
Err(mut err) => match self.token.kind {
100+
Err(mut err) => match self.normalized_token.kind {
101101
token::Ident(name, false)
102102
if name == kw::Underscore && self.look_ahead(1, |t| t == &token::Comma) =>
103103
{
104104
// Special-case handling of `foo(_, _, _)`
105105
err.emit();
106-
let sp = self.token.span;
107106
self.bump();
108-
Ok(self.mk_expr(sp, ExprKind::Err, AttrVec::new()))
107+
Ok(self.mk_expr(self.prev_token.span, ExprKind::Err, AttrVec::new()))
109108
}
110109
_ => Err(err),
111110
},
@@ -166,7 +165,7 @@ impl<'a> Parser<'a> {
166165
while let Some(op) = self.check_assoc_op() {
167166
// Adjust the span for interpolated LHS to point to the `$lhs` token
168167
// and not to what it refers to.
169-
let lhs_span = match self.unnormalized_prev_token.kind {
168+
let lhs_span = match self.prev_token.kind {
170169
TokenKind::Interpolated(..) => self.prev_span,
171170
_ => lhs.span,
172171
};
@@ -337,7 +336,7 @@ impl<'a> Parser<'a> {
337336
/// Also performs recovery for `and` / `or` which are mistaken for `&&` and `||` respectively.
338337
fn check_assoc_op(&self) -> Option<Spanned<AssocOp>> {
339338
Some(Spanned {
340-
node: match (AssocOp::from_token(&self.token), &self.token.kind) {
339+
node: match (AssocOp::from_token(&self.token), &self.normalized_token.kind) {
341340
(Some(op), _) => op,
342341
(None, token::Ident(sym::and, false)) => {
343342
self.error_bad_logical_op("and", "&&", "conjunction");
@@ -349,7 +348,7 @@ impl<'a> Parser<'a> {
349348
}
350349
_ => return None,
351350
},
352-
span: self.token.span,
351+
span: self.normalized_token.span,
353352
})
354353
}
355354

@@ -441,7 +440,7 @@ impl<'a> Parser<'a> {
441440
let attrs = self.parse_or_use_outer_attributes(attrs)?;
442441
let lo = self.token.span;
443442
// Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
444-
let (hi, ex) = match self.token.kind {
443+
let (hi, ex) = match self.normalized_token.kind {
445444
token::Not => self.parse_unary_expr(lo, UnOp::Not), // `!expr`
446445
token::Tilde => self.recover_tilde_expr(lo), // `~expr`
447446
token::BinOp(token::Minus) => self.parse_unary_expr(lo, UnOp::Neg), // `-expr`
@@ -527,7 +526,7 @@ impl<'a> Parser<'a> {
527526
) -> PResult<'a, (Span, P<Expr>)> {
528527
expr.map(|e| {
529528
(
530-
match self.unnormalized_prev_token.kind {
529+
match self.prev_token.kind {
531530
TokenKind::Interpolated(..) => self.prev_span,
532531
_ => e.span,
533532
},
@@ -708,7 +707,7 @@ impl<'a> Parser<'a> {
708707
}
709708

710709
fn parse_dot_suffix_expr(&mut self, lo: Span, base: P<Expr>) -> PResult<'a, P<Expr>> {
711-
match self.token.kind {
710+
match self.normalized_token.kind {
712711
token::Ident(..) => self.parse_dot_suffix(base, lo),
713712
token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => {
714713
Ok(self.parse_tuple_field_access_expr(lo, base, symbol, suffix))
@@ -777,8 +776,8 @@ impl<'a> Parser<'a> {
777776
field: Symbol,
778777
suffix: Option<Symbol>,
779778
) -> P<Expr> {
780-
let span = self.token.span;
781779
self.bump();
780+
let span = self.prev_token.span;
782781
let field = ExprKind::Field(base, Ident::new(field, span));
783782
self.expect_no_suffix(span, "a tuple index", suffix);
784783
self.mk_expr(lo.to(span), field, AttrVec::new())
@@ -802,7 +801,7 @@ impl<'a> Parser<'a> {
802801

803802
/// Assuming we have just parsed `.`, continue parsing into an expression.
804803
fn parse_dot_suffix(&mut self, self_arg: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> {
805-
if self.token.span.rust_2018() && self.eat_keyword(kw::Await) {
804+
if self.normalized_token.span.rust_2018() && self.eat_keyword(kw::Await) {
806805
return self.mk_await_expr(self_arg, lo);
807806
}
808807

@@ -916,7 +915,7 @@ impl<'a> Parser<'a> {
916915
// | ^ expected expression
917916
self.bump();
918917
Ok(self.mk_expr_err(self.token.span))
919-
} else if self.token.span.rust_2018() {
918+
} else if self.normalized_token.span.rust_2018() {
920919
// `Span::rust_2018()` is somewhat expensive; don't get it repeatedly.
921920
if self.check_keyword(kw::Async) {
922921
if self.is_async_block() {
@@ -1346,7 +1345,7 @@ impl<'a> Parser<'a> {
13461345
if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable };
13471346

13481347
let asyncness =
1349-
if self.token.span.rust_2018() { self.parse_asyncness() } else { Async::No };
1348+
if self.normalized_token.span.rust_2018() { self.parse_asyncness() } else { Async::No };
13501349
if asyncness.is_async() {
13511350
// Feature-gate `async ||` closures.
13521351
self.sess.gated_spans.gate(sym::async_closure, self.prev_span);
@@ -1560,9 +1559,8 @@ impl<'a> Parser<'a> {
15601559

15611560
fn eat_label(&mut self) -> Option<Label> {
15621561
self.token.lifetime().map(|ident| {
1563-
let span = self.token.span;
15641562
self.bump();
1565-
Label { ident: Ident::new(ident.name, span) }
1563+
Label { ident }
15661564
})
15671565
}
15681566

@@ -1704,7 +1702,7 @@ impl<'a> Parser<'a> {
17041702
fn is_try_block(&self) -> bool {
17051703
self.token.is_keyword(kw::Try) &&
17061704
self.look_ahead(1, |t| *t == token::OpenDelim(token::Brace)) &&
1707-
self.token.span.rust_2018() &&
1705+
self.normalized_token.span.rust_2018() &&
17081706
// Prevent `while try {} {}`, `if try {} {} else {}`, etc.
17091707
!self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL)
17101708
}
@@ -1854,13 +1852,12 @@ impl<'a> Parser<'a> {
18541852

18551853
/// Use in case of error after field-looking code: `S { foo: () with a }`.
18561854
fn find_struct_error_after_field_looking_code(&self) -> Option<Field> {
1857-
if let token::Ident(name, _) = self.token.kind {
1855+
if let token::Ident(name, _) = self.normalized_token.kind {
18581856
if !self.token.is_reserved_ident() && self.look_ahead(1, |t| *t == token::Colon) {
1859-
let span = self.token.span;
18601857
return Some(ast::Field {
1861-
ident: Ident::new(name, span),
1862-
span,
1863-
expr: self.mk_expr_err(span),
1858+
ident: Ident::new(name, self.normalized_token.span),
1859+
span: self.token.span,
1860+
expr: self.mk_expr_err(self.token.span),
18641861
is_shorthand: false,
18651862
attrs: AttrVec::new(),
18661863
id: DUMMY_NODE_ID,

Diff for: ‎src/librustc_parse/parser/item.rs

+6-8
Original file line numberDiff line numberDiff line change
@@ -747,11 +747,10 @@ impl<'a> Parser<'a> {
747747
}
748748

749749
fn parse_ident_or_underscore(&mut self) -> PResult<'a, ast::Ident> {
750-
match self.token.kind {
750+
match self.normalized_token.kind {
751751
token::Ident(name @ kw::Underscore, false) => {
752-
let span = self.token.span;
753752
self.bump();
754-
Ok(Ident::new(name, span))
753+
Ok(Ident::new(name, self.normalized_prev_token.span))
755754
}
756755
_ => self.parse_ident(),
757756
}
@@ -1545,7 +1544,7 @@ impl<'a> Parser<'a> {
15451544

15461545
let is_name_required = match self.token.kind {
15471546
token::DotDotDot => false,
1548-
_ => req_name(&self.token),
1547+
_ => req_name(&self.normalized_token),
15491548
};
15501549
let (pat, ty) = if is_name_required || self.is_named_param() {
15511550
debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required);
@@ -1611,12 +1610,11 @@ impl<'a> Parser<'a> {
16111610
fn parse_self_param(&mut self) -> PResult<'a, Option<Param>> {
16121611
// Extract an identifier *after* having confirmed that the token is one.
16131612
let expect_self_ident = |this: &mut Self| {
1614-
match this.token.kind {
1613+
match this.normalized_token.kind {
16151614
// Preserve hygienic context.
16161615
token::Ident(name, _) => {
1617-
let span = this.token.span;
16181616
this.bump();
1619-
Ident::new(name, span)
1617+
Ident::new(name, this.normalized_prev_token.span)
16201618
}
16211619
_ => unreachable!(),
16221620
}
@@ -1653,7 +1651,7 @@ impl<'a> Parser<'a> {
16531651
// Only a limited set of initial token sequences is considered `self` parameters; anything
16541652
// else is parsed as a normal function parameter list, so some lookahead is required.
16551653
let eself_lo = self.token.span;
1656-
let (eself, eself_ident, eself_hi) = match self.token.kind {
1654+
let (eself, eself_ident, eself_hi) = match self.normalized_token.kind {
16571655
token::BinOp(token::And) => {
16581656
let eself = if is_isolated_self(self, 1) {
16591657
// `&self`

Diff for: ‎src/librustc_parse/parser/mod.rs

+23-25
Original file line numberDiff line numberDiff line change
@@ -86,23 +86,22 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
8686
#[derive(Clone)]
8787
pub struct Parser<'a> {
8888
pub sess: &'a ParseSess,
89+
/// The current non-normalized token.
90+
pub token: Token,
8991
/// The current normalized token.
9092
/// "Normalized" means that some interpolated tokens
9193
/// (`$i: ident` and `$l: lifetime` meta-variables) are replaced
9294
/// with non-interpolated identifier and lifetime tokens they refer to.
93-
/// Use span from this token if you need an isolated span.
94-
pub token: Token,
95-
/// The current non-normalized token if it's different from `token`.
96-
/// Use span from this token if you need to concatenate it with some neighbouring spans.
97-
unnormalized_token: Token,
95+
/// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
96+
/// this also includes edition checks for edition-specific keyword identifiers.
97+
pub normalized_token: Token,
98+
/// The previous non-normalized token.
99+
pub prev_token: Token,
98100
/// The previous normalized token.
99-
/// Use span from this token if you need an isolated span.
100-
prev_token: Token,
101-
/// The previous non-normalized token if it's different from `prev_token`.
102-
/// Use span from this token if you need to concatenate it with some neighbouring spans.
103-
unnormalized_prev_token: Token,
104-
/// Equivalent to `unnormalized_prev_token.span`.
105-
/// FIXME: Remove in favor of `(unnormalized_)prev_token.span`.
101+
/// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
102+
/// this also includes edition checks for edition-specific keyword identifiers.
103+
pub normalized_prev_token: Token,
104+
/// FIXME: Remove in favor of the equivalent `prev_token.span`.
106105
pub prev_span: Span,
107106
restrictions: Restrictions,
108107
/// Used to determine the path to externally loaded source files.
@@ -376,9 +375,9 @@ impl<'a> Parser<'a> {
376375
let mut parser = Parser {
377376
sess,
378377
token: Token::dummy(),
379-
unnormalized_token: Token::dummy(),
378+
normalized_token: Token::dummy(),
380379
prev_token: Token::dummy(),
381-
unnormalized_prev_token: Token::dummy(),
380+
normalized_prev_token: Token::dummy(),
382381
prev_span: DUMMY_SP,
383382
restrictions: Restrictions::empty(),
384383
recurse_into_file_modules,
@@ -483,7 +482,7 @@ impl<'a> Parser<'a> {
483482
}
484483

485484
fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
486-
match self.token.kind {
485+
match self.normalized_token.kind {
487486
token::Ident(name, _) => {
488487
if self.token.is_reserved_ident() {
489488
let mut err = self.expected_ident_found();
@@ -493,9 +492,8 @@ impl<'a> Parser<'a> {
493492
return Err(err);
494493
}
495494
}
496-
let span = self.token.span;
497495
self.bump();
498-
Ok(Ident::new(name, span))
496+
Ok(Ident::new(name, self.normalized_prev_token.span))
499497
}
500498
_ => Err(match self.prev_token.kind {
501499
TokenKind::DocComment(..) => {
@@ -825,16 +823,16 @@ impl<'a> Parser<'a> {
825823
// tokens are replaced with usual identifier and lifetime tokens,
826824
// so the former are never encountered during normal parsing.
827825
crate fn set_token(&mut self, token: Token) {
828-
self.unnormalized_token = token;
829-
self.token = match &self.unnormalized_token.kind {
826+
self.token = token;
827+
self.normalized_token = match &self.token.kind {
830828
token::Interpolated(nt) => match **nt {
831829
token::NtIdent(ident, is_raw) => {
832830
Token::new(token::Ident(ident.name, is_raw), ident.span)
833831
}
834832
token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span),
835-
_ => self.unnormalized_token.clone(),
833+
_ => self.token.clone(),
836834
},
837-
_ => self.unnormalized_token.clone(),
835+
_ => self.token.clone(),
838836
}
839837
}
840838

@@ -848,19 +846,19 @@ impl<'a> Parser<'a> {
848846

849847
// Update the current and previous tokens.
850848
self.prev_token = self.token.take();
851-
self.unnormalized_prev_token = self.unnormalized_token.take();
849+
self.normalized_prev_token = self.normalized_token.take();
852850
self.set_token(next_token);
853851

854852
// Update fields derived from the previous token.
855-
self.prev_span = self.unnormalized_prev_token.span;
853+
self.prev_span = self.prev_token.span;
856854

857855
// Diagnostics.
858856
self.expected_tokens.clear();
859857
}
860858

861859
/// Advance the parser by one token.
862860
pub fn bump(&mut self) {
863-
let next_token = self.next_tok(self.unnormalized_token.span);
861+
let next_token = self.next_tok(self.token.span);
864862
self.bump_with(next_token);
865863
}
866864

@@ -891,7 +889,7 @@ impl<'a> Parser<'a> {
891889
/// Parses asyncness: `async` or nothing.
892890
fn parse_asyncness(&mut self) -> Async {
893891
if self.eat_keyword(kw::Async) {
894-
let span = self.prev_span;
892+
let span = self.normalized_prev_token.span;
895893
Async::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
896894
} else {
897895
Async::No

Diff for: ‎src/librustc_parse/parser/path.rs

+3-4
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ impl<'a> Parser<'a> {
134134
path
135135
});
136136

137-
let lo = self.unnormalized_token.span;
137+
let lo = self.token.span;
138138
let mut segments = Vec::new();
139139
let mod_sep_ctxt = self.token.span.ctxt();
140140
if self.eat(&token::ModSep) {
@@ -238,11 +238,10 @@ impl<'a> Parser<'a> {
238238
}
239239

240240
pub(super) fn parse_path_segment_ident(&mut self) -> PResult<'a, Ident> {
241-
match self.token.kind {
241+
match self.normalized_token.kind {
242242
token::Ident(name, _) if name.is_path_segment_keyword() => {
243-
let span = self.token.span;
244243
self.bump();
245-
Ok(Ident::new(name, span))
244+
Ok(Ident::new(name, self.normalized_prev_token.span))
246245
}
247246
_ => self.parse_ident(),
248247
}

0 commit comments

Comments
 (0)