Skip to content

Commit decb0c9

Browse files
matthiaskrgrmatthiaskrgr
authored
Rollup merge of #137902 - nnethercote:ast-lexer-TokenKind, r=compiler-errors
Make `ast::TokenKind` more like `lexer::TokenKind` This is step 2 of rust-lang/compiler-team#831. r? `@spastorino`
2 parents 70b9968 + 53167c0 commit decb0c9

File tree

24 files changed

+396
-353
lines changed

24 files changed

+396
-353
lines changed

‎compiler/rustc_ast/src/token.rs

+170-126
Large diffs are not rendered by default.

‎compiler/rustc_ast/src/tokenstream.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -651,7 +651,7 @@ impl TokenStream {
651651
if attr_style == AttrStyle::Inner {
652652
vec![
653653
TokenTree::token_joint(token::Pound, span),
654-
TokenTree::token_joint_hidden(token::Not, span),
654+
TokenTree::token_joint_hidden(token::Bang, span),
655655
body,
656656
]
657657
} else {

‎compiler/rustc_ast/src/util/parser.rs

+21-21
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use rustc_span::kw;
22

33
use crate::ast::{self, BinOpKind, RangeLimits};
4-
use crate::token::{self, BinOpToken, Token};
4+
use crate::token::{self, Token};
55

66
/// Associative operator.
77
#[derive(Copy, Clone, PartialEq, Debug)]
@@ -34,26 +34,26 @@ impl AssocOp {
3434
use AssocOp::*;
3535
match t.kind {
3636
token::Eq => Some(Assign),
37-
token::BinOp(BinOpToken::Plus) => Some(Binary(BinOpKind::Add)),
38-
token::BinOp(BinOpToken::Minus) => Some(Binary(BinOpKind::Sub)),
39-
token::BinOp(BinOpToken::Star) => Some(Binary(BinOpKind::Mul)),
40-
token::BinOp(BinOpToken::Slash) => Some(Binary(BinOpKind::Div)),
41-
token::BinOp(BinOpToken::Percent) => Some(Binary(BinOpKind::Rem)),
42-
token::BinOp(BinOpToken::Caret) => Some(Binary(BinOpKind::BitXor)),
43-
token::BinOp(BinOpToken::And) => Some(Binary(BinOpKind::BitAnd)),
44-
token::BinOp(BinOpToken::Or) => Some(Binary(BinOpKind::BitOr)),
45-
token::BinOp(BinOpToken::Shl) => Some(Binary(BinOpKind::Shl)),
46-
token::BinOp(BinOpToken::Shr) => Some(Binary(BinOpKind::Shr)),
47-
token::BinOpEq(BinOpToken::Plus) => Some(AssignOp(BinOpKind::Add)),
48-
token::BinOpEq(BinOpToken::Minus) => Some(AssignOp(BinOpKind::Sub)),
49-
token::BinOpEq(BinOpToken::Star) => Some(AssignOp(BinOpKind::Mul)),
50-
token::BinOpEq(BinOpToken::Slash) => Some(AssignOp(BinOpKind::Div)),
51-
token::BinOpEq(BinOpToken::Percent) => Some(AssignOp(BinOpKind::Rem)),
52-
token::BinOpEq(BinOpToken::Caret) => Some(AssignOp(BinOpKind::BitXor)),
53-
token::BinOpEq(BinOpToken::And) => Some(AssignOp(BinOpKind::BitAnd)),
54-
token::BinOpEq(BinOpToken::Or) => Some(AssignOp(BinOpKind::BitOr)),
55-
token::BinOpEq(BinOpToken::Shl) => Some(AssignOp(BinOpKind::Shl)),
56-
token::BinOpEq(BinOpToken::Shr) => Some(AssignOp(BinOpKind::Shr)),
37+
token::Plus => Some(Binary(BinOpKind::Add)),
38+
token::Minus => Some(Binary(BinOpKind::Sub)),
39+
token::Star => Some(Binary(BinOpKind::Mul)),
40+
token::Slash => Some(Binary(BinOpKind::Div)),
41+
token::Percent => Some(Binary(BinOpKind::Rem)),
42+
token::Caret => Some(Binary(BinOpKind::BitXor)),
43+
token::And => Some(Binary(BinOpKind::BitAnd)),
44+
token::Or => Some(Binary(BinOpKind::BitOr)),
45+
token::Shl => Some(Binary(BinOpKind::Shl)),
46+
token::Shr => Some(Binary(BinOpKind::Shr)),
47+
token::PlusEq => Some(AssignOp(BinOpKind::Add)),
48+
token::MinusEq => Some(AssignOp(BinOpKind::Sub)),
49+
token::StarEq => Some(AssignOp(BinOpKind::Mul)),
50+
token::SlashEq => Some(AssignOp(BinOpKind::Div)),
51+
token::PercentEq => Some(AssignOp(BinOpKind::Rem)),
52+
token::CaretEq => Some(AssignOp(BinOpKind::BitXor)),
53+
token::AndEq => Some(AssignOp(BinOpKind::BitAnd)),
54+
token::OrEq => Some(AssignOp(BinOpKind::BitOr)),
55+
token::ShlEq => Some(AssignOp(BinOpKind::Shl)),
56+
token::ShrEq => Some(AssignOp(BinOpKind::Shr)),
5757
token::Lt => Some(Binary(BinOpKind::Lt)),
5858
token::Le => Some(Binary(BinOpKind::Le)),
5959
token::Ge => Some(Binary(BinOpKind::Ge)),

‎compiler/rustc_ast_pretty/src/pprust/state.rs

+23-22
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,7 @@ use std::sync::Arc;
1111

1212
use rustc_ast::attr::AttrIdGenerator;
1313
use rustc_ast::ptr::P;
14-
use rustc_ast::token::{
15-
self, BinOpToken, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind,
16-
};
14+
use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
1715
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
1816
use rustc_ast::util::classify;
1917
use rustc_ast::util::comments::{Comment, CommentStyle};
@@ -319,7 +317,7 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
319317
(tt1, Tok(Token { kind: Comma | Semi | Dot, .. }, _)) if !is_punct(tt1) => false,
320318

321319
// IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if`
322-
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Not, .. }, _))
320+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Bang, .. }, _))
323321
if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, IdentIsRaw::Yes) =>
324322
{
325323
false
@@ -344,21 +342,6 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
344342
}
345343
}
346344

347-
fn binop_to_string(op: BinOpToken) -> &'static str {
348-
match op {
349-
token::Plus => "+",
350-
token::Minus => "-",
351-
token::Star => "*",
352-
token::Slash => "/",
353-
token::Percent => "%",
354-
token::Caret => "^",
355-
token::And => "&",
356-
token::Or => "|",
357-
token::Shl => "<<",
358-
token::Shr => ">>",
359-
}
360-
}
361-
362345
pub fn doc_comment_to_string(
363346
comment_kind: CommentKind,
364347
attr_style: ast::AttrStyle,
@@ -913,12 +896,30 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
913896
token::Ne => "!=".into(),
914897
token::Ge => ">=".into(),
915898
token::Gt => ">".into(),
916-
token::Not => "!".into(),
899+
token::Bang => "!".into(),
917900
token::Tilde => "~".into(),
918901
token::OrOr => "||".into(),
919902
token::AndAnd => "&&".into(),
920-
token::BinOp(op) => binop_to_string(op).into(),
921-
token::BinOpEq(op) => format!("{}=", binop_to_string(op)).into(),
903+
token::Plus => "+".into(),
904+
token::Minus => "-".into(),
905+
token::Star => "*".into(),
906+
token::Slash => "/".into(),
907+
token::Percent => "%".into(),
908+
token::Caret => "^".into(),
909+
token::And => "&".into(),
910+
token::Or => "|".into(),
911+
token::Shl => "<<".into(),
912+
token::Shr => ">>".into(),
913+
token::PlusEq => "+=".into(),
914+
token::MinusEq => "-=".into(),
915+
token::StarEq => "*=".into(),
916+
token::SlashEq => "/=".into(),
917+
token::PercentEq => "%=".into(),
918+
token::CaretEq => "^=".into(),
919+
token::AndEq => "&=".into(),
920+
token::OrEq => "|=".into(),
921+
token::ShlEq => "<<=".into(),
922+
token::ShrEq => ">>=".into(),
922923

923924
/* Structural symbols */
924925
token::At => "@".into(),

‎compiler/rustc_expand/src/config.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ impl<'a> StripUnconfigured<'a> {
328328

329329
// For inner attributes, we do the same thing for the `!` in `#![attr]`.
330330
let mut trees = if cfg_attr.style == AttrStyle::Inner {
331-
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _)) =
331+
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Bang, .. }, _)) =
332332
orig_trees.next()
333333
else {
334334
panic!("Bad tokens for attribute {cfg_attr:?}");

‎compiler/rustc_expand/src/mbe/macro_check.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ fn check_nested_occurrences(
432432
}
433433
(
434434
NestedMacroState::MacroRules,
435-
&TokenTree::Token(Token { kind: TokenKind::Not, .. }),
435+
&TokenTree::Token(Token { kind: TokenKind::Bang, .. }),
436436
) => {
437437
state = NestedMacroState::MacroRulesNot;
438438
}

‎compiler/rustc_expand/src/mbe/macro_rules.rs

+6-6
Original file line numberDiff line numberDiff line change
@@ -690,7 +690,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
690690
&& let TokenKind::Ident(ident, _) = ident.kind
691691
&& ident == sym::compile_error
692692
&& let mbe::TokenTree::Token(bang) = bang
693-
&& let TokenKind::Not = bang.kind
693+
&& let TokenKind::Bang = bang.kind
694694
&& let mbe::TokenTree::Delimited(.., del) = args
695695
&& !del.delim.skip()
696696
{
@@ -1135,7 +1135,7 @@ fn check_matcher_core<'tt>(
11351135
&& matches!(kind, NonterminalKind::Pat(PatParam { inferred: true }))
11361136
&& matches!(
11371137
next_token,
1138-
TokenTree::Token(token) if *token == BinOp(token::BinOpToken::Or)
1138+
TokenTree::Token(token) if *token == token::Or
11391139
)
11401140
{
11411141
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
@@ -1177,7 +1177,7 @@ fn check_matcher_core<'tt>(
11771177

11781178
if kind == NonterminalKind::Pat(PatWithOr)
11791179
&& sess.psess.edition.at_least_rust_2021()
1180-
&& next_token.is_token(&BinOp(token::BinOpToken::Or))
1180+
&& next_token.is_token(&token::Or)
11811181
{
11821182
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
11831183
span,
@@ -1296,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
12961296
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
12971297
match tok {
12981298
TokenTree::Token(token) => match token.kind {
1299-
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
1299+
FatArrow | Comma | Eq | Or => IsInFollow::Yes,
13001300
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
13011301
IsInFollow::Yes
13021302
}
@@ -1332,9 +1332,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13321332
| Colon
13331333
| Eq
13341334
| Gt
1335-
| BinOp(token::Shr)
1335+
| Shr
13361336
| Semi
1337-
| BinOp(token::Or) => IsInFollow::Yes,
1337+
| Or => IsInFollow::Yes,
13381338
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
13391339
IsInFollow::Yes
13401340
}

‎compiler/rustc_expand/src/mbe/quoted.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -302,8 +302,8 @@ fn parse_tree<'a>(
302302
/// `None`.
303303
fn kleene_op(token: &Token) -> Option<KleeneOp> {
304304
match token.kind {
305-
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
306-
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
305+
token::Star => Some(KleeneOp::ZeroOrMore),
306+
token::Plus => Some(KleeneOp::OneOrMore),
307307
token::Question => Some(KleeneOp::ZeroOrOne),
308308
_ => None,
309309
}

‎compiler/rustc_expand/src/proc_macro_server.rs

+33-38
Original file line numberDiff line numberDiff line change
@@ -180,28 +180,28 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
180180
Gt => op(">"),
181181
AndAnd => op("&&"),
182182
OrOr => op("||"),
183-
Not => op("!"),
183+
Bang => op("!"),
184184
Tilde => op("~"),
185-
BinOp(Plus) => op("+"),
186-
BinOp(Minus) => op("-"),
187-
BinOp(Star) => op("*"),
188-
BinOp(Slash) => op("/"),
189-
BinOp(Percent) => op("%"),
190-
BinOp(Caret) => op("^"),
191-
BinOp(And) => op("&"),
192-
BinOp(Or) => op("|"),
193-
BinOp(Shl) => op("<<"),
194-
BinOp(Shr) => op(">>"),
195-
BinOpEq(Plus) => op("+="),
196-
BinOpEq(Minus) => op("-="),
197-
BinOpEq(Star) => op("*="),
198-
BinOpEq(Slash) => op("/="),
199-
BinOpEq(Percent) => op("%="),
200-
BinOpEq(Caret) => op("^="),
201-
BinOpEq(And) => op("&="),
202-
BinOpEq(Or) => op("|="),
203-
BinOpEq(Shl) => op("<<="),
204-
BinOpEq(Shr) => op(">>="),
185+
Plus => op("+"),
186+
Minus => op("-"),
187+
Star => op("*"),
188+
Slash => op("/"),
189+
Percent => op("%"),
190+
Caret => op("^"),
191+
And => op("&"),
192+
Or => op("|"),
193+
Shl => op("<<"),
194+
Shr => op(">>"),
195+
PlusEq => op("+="),
196+
MinusEq => op("-="),
197+
StarEq => op("*="),
198+
SlashEq => op("/="),
199+
PercentEq => op("%="),
200+
CaretEq => op("^="),
201+
AndEq => op("&="),
202+
OrEq => op("|="),
203+
ShlEq => op("<<="),
204+
ShrEq => op(">>="),
205205
At => op("@"),
206206
Dot => op("."),
207207
DotDot => op(".."),
@@ -322,16 +322,16 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
322322
b'=' => Eq,
323323
b'<' => Lt,
324324
b'>' => Gt,
325-
b'!' => Not,
325+
b'!' => Bang,
326326
b'~' => Tilde,
327-
b'+' => BinOp(Plus),
328-
b'-' => BinOp(Minus),
329-
b'*' => BinOp(Star),
330-
b'/' => BinOp(Slash),
331-
b'%' => BinOp(Percent),
332-
b'^' => BinOp(Caret),
333-
b'&' => BinOp(And),
334-
b'|' => BinOp(Or),
327+
b'+' => Plus,
328+
b'-' => Minus,
329+
b'*' => Star,
330+
b'/' => Slash,
331+
b'%' => Percent,
332+
b'^' => Caret,
333+
b'&' => And,
334+
b'|' => Or,
335335
b'@' => At,
336336
b'.' => Dot,
337337
b',' => Comma,
@@ -372,10 +372,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
372372
suffix,
373373
span,
374374
}) if symbol.as_str().starts_with('-') => {
375-
let minus = BinOp(BinOpToken::Minus);
376375
let symbol = Symbol::intern(&symbol.as_str()[1..]);
377376
let integer = TokenKind::lit(token::Integer, symbol, suffix);
378-
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
377+
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
379378
let b = tokenstream::TokenTree::token_alone(integer, span);
380379
smallvec![a, b]
381380
}
@@ -385,10 +384,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
385384
suffix,
386385
span,
387386
}) if symbol.as_str().starts_with('-') => {
388-
let minus = BinOp(BinOpToken::Minus);
389387
let symbol = Symbol::intern(&symbol.as_str()[1..]);
390388
let float = TokenKind::lit(token::Float, symbol, suffix);
391-
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
389+
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
392390
let b = tokenstream::TokenTree::token_alone(float, span);
393391
smallvec![a, b]
394392
}
@@ -599,10 +597,7 @@ impl server::TokenStream for Rustc<'_, '_> {
599597
Ok(Self::TokenStream::from_iter([
600598
// FIXME: The span of the `-` token is lost when
601599
// parsing, so we cannot faithfully recover it here.
602-
tokenstream::TokenTree::token_joint_hidden(
603-
token::BinOp(token::Minus),
604-
e.span,
605-
),
600+
tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span),
606601
tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
607602
]))
608603
}

‎compiler/rustc_parse/src/lexer/mod.rs

+9-9
Original file line numberDiff line numberDiff line change
@@ -384,17 +384,17 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
384384
rustc_lexer::TokenKind::Colon => token::Colon,
385385
rustc_lexer::TokenKind::Dollar => token::Dollar,
386386
rustc_lexer::TokenKind::Eq => token::Eq,
387-
rustc_lexer::TokenKind::Bang => token::Not,
387+
rustc_lexer::TokenKind::Bang => token::Bang,
388388
rustc_lexer::TokenKind::Lt => token::Lt,
389389
rustc_lexer::TokenKind::Gt => token::Gt,
390-
rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus),
391-
rustc_lexer::TokenKind::And => token::BinOp(token::And),
392-
rustc_lexer::TokenKind::Or => token::BinOp(token::Or),
393-
rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus),
394-
rustc_lexer::TokenKind::Star => token::BinOp(token::Star),
395-
rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash),
396-
rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret),
397-
rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
390+
rustc_lexer::TokenKind::Minus => token::Minus,
391+
rustc_lexer::TokenKind::And => token::And,
392+
rustc_lexer::TokenKind::Or => token::Or,
393+
rustc_lexer::TokenKind::Plus => token::Plus,
394+
rustc_lexer::TokenKind::Star => token::Star,
395+
rustc_lexer::TokenKind::Slash => token::Slash,
396+
rustc_lexer::TokenKind::Caret => token::Caret,
397+
rustc_lexer::TokenKind::Percent => token::Percent,
398398

399399
rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => {
400400
// Don't emit diagnostics for sequences of the same invalid token

‎compiler/rustc_parse/src/lexer/unicode_chars.rs

+6-6
Original file line numberDiff line numberDiff line change
@@ -308,11 +308,11 @@ pub(super) static UNICODE_ARRAY: &[(char, &str, &str)] = &[
308308
const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
309309
(" ", "Space", None),
310310
("_", "Underscore", Some(token::Ident(kw::Underscore, token::IdentIsRaw::No))),
311-
("-", "Minus/Hyphen", Some(token::BinOp(token::Minus))),
311+
("-", "Minus/Hyphen", Some(token::Minus)),
312312
(",", "Comma", Some(token::Comma)),
313313
(";", "Semicolon", Some(token::Semi)),
314314
(":", "Colon", Some(token::Colon)),
315-
("!", "Exclamation Mark", Some(token::Not)),
315+
("!", "Exclamation Mark", Some(token::Bang)),
316316
("?", "Question Mark", Some(token::Question)),
317317
(".", "Period", Some(token::Dot)),
318318
("(", "Left Parenthesis", Some(token::OpenDelim(Delimiter::Parenthesis))),
@@ -321,11 +321,11 @@ const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
321321
("]", "Right Square Bracket", Some(token::CloseDelim(Delimiter::Bracket))),
322322
("{", "Left Curly Brace", Some(token::OpenDelim(Delimiter::Brace))),
323323
("}", "Right Curly Brace", Some(token::CloseDelim(Delimiter::Brace))),
324-
("*", "Asterisk", Some(token::BinOp(token::Star))),
325-
("/", "Slash", Some(token::BinOp(token::Slash))),
324+
("*", "Asterisk", Some(token::Star)),
325+
("/", "Slash", Some(token::Slash)),
326326
("\\", "Backslash", None),
327-
("&", "Ampersand", Some(token::BinOp(token::And))),
328-
("+", "Plus Sign", Some(token::BinOp(token::Plus))),
327+
("&", "Ampersand", Some(token::And)),
328+
("+", "Plus Sign", Some(token::Plus)),
329329
("<", "Less-Than Sign", Some(token::Lt)),
330330
("=", "Equals Sign", Some(token::Eq)),
331331
("==", "Double Equals Sign", Some(token::EqEq)),

0 commit comments

Comments
 (0)