Skip to content

Commit bebd57a

Browse files
borsbors
committed
Auto merge of #102944 - nnethercote:ast-Lit-third-time-lucky, r=petrochenkov
Use `token::Lit` in `ast::ExprKind::Lit`. Instead of `ast::Lit`. Literal lowering now happens at two different times. Expression literals are lowered when HIR is crated. Attribute literals are lowered during parsing. r? `@petrochenkov`
2 parents e9493d6 + 358a603 commit bebd57a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+786
-507
lines changed

‎compiler/rustc_ast/src/ast.rs

+11-13
Original file line numberDiff line numberDiff line change
@@ -1332,7 +1332,7 @@ pub enum ExprKind {
13321332
/// A unary operation (e.g., `!x`, `*x`).
13331333
Unary(UnOp, P<Expr>),
13341334
/// A literal (e.g., `1`, `"foo"`).
1335-
Lit(Lit),
1335+
Lit(token::Lit),
13361336
/// A cast (e.g., `foo as f64`).
13371337
Cast(P<Expr>, P<Ty>),
13381338
/// A type ascription (e.g., `42: usize`).
@@ -1698,16 +1698,12 @@ pub struct StrLit {
16981698
}
16991699

17001700
impl StrLit {
1701-
pub fn as_lit(&self) -> Lit {
1701+
pub fn as_token_lit(&self) -> token::Lit {
17021702
let token_kind = match self.style {
17031703
StrStyle::Cooked => token::Str,
17041704
StrStyle::Raw(n) => token::StrRaw(n),
17051705
};
1706-
Lit {
1707-
token_lit: token::Lit::new(token_kind, self.symbol, self.suffix),
1708-
span: self.span,
1709-
kind: LitKind::Str(self.symbol_unescaped, self.style),
1710-
}
1706+
token::Lit::new(token_kind, self.symbol, self.suffix)
17111707
}
17121708
}
17131709

@@ -1733,9 +1729,10 @@ pub enum LitFloatType {
17331729
Unsuffixed,
17341730
}
17351731

1736-
/// Literal kind.
1737-
///
1738-
/// E.g., `"foo"`, `42`, `12.34`, or `bool`.
1732+
/// Note that the entire literal (including the suffix) is considered when
1733+
/// deciding the `LitKind`. This means that float literals like `1f32` are
1734+
/// classified by this type as `Float`. This is different to `token::LitKind`
1735+
/// which does *not* consider the suffix.
17391736
#[derive(Clone, Encodable, Decodable, Debug, Hash, Eq, PartialEq, HashStable_Generic)]
17401737
pub enum LitKind {
17411738
/// A string literal (`"foo"`). The symbol is unescaped, and so may differ
@@ -1749,10 +1746,11 @@ pub enum LitKind {
17491746
Char(char),
17501747
/// An integer literal (`1`).
17511748
Int(u128, LitIntType),
1752-
/// A float literal (`1f64` or `1E10f64`). Stored as a symbol rather than
1753-
/// `f64` so that `LitKind` can impl `Eq` and `Hash`.
1749+
/// A float literal (`1.0`, `1f64` or `1E10f64`). The pre-suffix part is
1750+
/// stored as a symbol rather than `f64` so that `LitKind` can impl `Eq`
1751+
/// and `Hash`.
17541752
Float(Symbol, LitFloatType),
1755-
/// A boolean literal.
1753+
/// A boolean literal (`true`, `false`).
17561754
Bool(bool),
17571755
/// Placeholder for a literal that wasn't well-formed in some way.
17581756
Err,

‎compiler/rustc_ast/src/attr/mod.rs

+7-5
Original file line numberDiff line numberDiff line change
@@ -533,7 +533,7 @@ impl MetaItemKind {
533533
MetaItemKind::NameValue(lit) => {
534534
let expr = P(ast::Expr {
535535
id: ast::DUMMY_NODE_ID,
536-
kind: ast::ExprKind::Lit(lit.clone()),
536+
kind: ast::ExprKind::Lit(lit.token_lit.clone()),
537537
span: lit.span,
538538
attrs: ast::AttrVec::new(),
539539
tokens: None,
@@ -605,7 +605,7 @@ impl MetaItemKind {
605605
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
606606
}
607607
Some(TokenTree::Token(token, _)) => {
608-
Lit::from_token(&token).ok().map(MetaItemKind::NameValue)
608+
Lit::from_token(&token).map(MetaItemKind::NameValue)
609609
}
610610
_ => None,
611611
}
@@ -618,8 +618,10 @@ impl MetaItemKind {
618618
MetaItemKind::list_from_tokens(tokens.clone())
619619
}
620620
MacArgs::Delimited(..) => None,
621-
MacArgs::Eq(_, MacArgsEq::Ast(expr)) => match &expr.kind {
622-
ast::ExprKind::Lit(lit) => Some(MetaItemKind::NameValue(lit.clone())),
621+
MacArgs::Eq(_, MacArgsEq::Ast(expr)) => match expr.kind {
622+
ast::ExprKind::Lit(token_lit) => Some(MetaItemKind::NameValue(
623+
Lit::from_token_lit(token_lit, expr.span).expect("token_lit in from_mac_args"),
624+
)),
623625
_ => None,
624626
},
625627
MacArgs::Eq(_, MacArgsEq::Hir(lit)) => Some(MetaItemKind::NameValue(lit.clone())),
@@ -668,7 +670,7 @@ impl NestedMetaItem {
668670
{
669671
match tokens.peek() {
670672
Some(TokenTree::Token(token, _))
671-
if let Ok(lit) = Lit::from_token(token) =>
673+
if let Some(lit) = Lit::from_token(token) =>
672674
{
673675
tokens.next();
674676
return Some(NestedMetaItem::Literal(lit));

‎compiler/rustc_ast/src/token.rs

+42-8
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,17 @@ pub enum Delimiter {
5959
Invisible,
6060
}
6161

62+
// Note that the suffix is *not* considered when deciding the `LitKind` in this
63+
// type. This means that float literals like `1f32` are classified by this type
64+
// as `Int`. Only upon conversion to `ast::LitKind` will such a literal be
65+
// given the `Float` kind.
6266
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
6367
pub enum LitKind {
6468
Bool, // AST only, must never appear in a `Token`
6569
Byte,
6670
Char,
67-
Integer,
68-
Float,
71+
Integer, // e.g. `1`, `1u8`, `1f32`
72+
Float, // e.g. `1.`, `1.0`, `1e3f32`
6973
Str,
7074
StrRaw(u8), // raw string delimited by `n` hash symbols
7175
ByteStr,
@@ -81,6 +85,42 @@ pub struct Lit {
8185
pub suffix: Option<Symbol>,
8286
}
8387

88+
impl Lit {
89+
pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
90+
Lit { kind, symbol, suffix }
91+
}
92+
93+
/// Returns `true` if this is semantically a float literal. This includes
94+
/// ones like `1f32` that have an `Integer` kind but a float suffix.
95+
pub fn is_semantic_float(&self) -> bool {
96+
match self.kind {
97+
LitKind::Float => true,
98+
LitKind::Integer => match self.suffix {
99+
Some(sym) => sym == sym::f32 || sym == sym::f64,
100+
None => false,
101+
},
102+
_ => false,
103+
}
104+
}
105+
106+
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
107+
pub fn from_token(token: &Token) -> Option<Lit> {
108+
match token.uninterpolate().kind {
109+
Ident(name, false) if name.is_bool_lit() => {
110+
Some(Lit::new(Bool, name, None))
111+
}
112+
Literal(token_lit) => Some(token_lit),
113+
Interpolated(ref nt)
114+
if let NtExpr(expr) | NtLiteral(expr) = &**nt
115+
&& let ast::ExprKind::Lit(token_lit) = expr.kind =>
116+
{
117+
Some(token_lit.clone())
118+
}
119+
_ => None,
120+
}
121+
}
122+
}
123+
84124
impl fmt::Display for Lit {
85125
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
86126
let Lit { kind, symbol, suffix } = *self;
@@ -139,12 +179,6 @@ impl LitKind {
139179
}
140180
}
141181

142-
impl Lit {
143-
pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
144-
Lit { kind, symbol, suffix }
145-
}
146-
}
147-
148182
pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
149183
let ident_token = Token::new(Ident(name, is_raw), span);
150184

‎compiler/rustc_ast/src/util/literal.rs

+5-22
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ use rustc_span::symbol::{kw, sym, Symbol};
88
use rustc_span::Span;
99
use std::ascii;
1010

11+
#[derive(Debug)]
1112
pub enum LitError {
12-
NotLiteral,
1313
LexerError,
1414
InvalidSuffix,
1515
InvalidIntSuffix,
@@ -202,27 +202,10 @@ impl Lit {
202202
Ok(Lit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
203203
}
204204

205-
/// Converts arbitrary token into an AST literal.
206-
///
207-
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
208-
pub fn from_token(token: &Token) -> Result<Lit, LitError> {
209-
let lit = match token.uninterpolate().kind {
210-
token::Ident(name, false) if name.is_bool_lit() => {
211-
token::Lit::new(token::Bool, name, None)
212-
}
213-
token::Literal(lit) => lit,
214-
token::Interpolated(ref nt) => {
215-
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt
216-
&& let ast::ExprKind::Lit(lit) = &expr.kind
217-
{
218-
return Ok(lit.clone());
219-
}
220-
return Err(LitError::NotLiteral);
221-
}
222-
_ => return Err(LitError::NotLiteral),
223-
};
224-
225-
Lit::from_token_lit(lit, token.span)
205+
/// Converts an arbitrary token into an AST literal.
206+
pub fn from_token(token: &Token) -> Option<Lit> {
207+
token::Lit::from_token(token)
208+
.and_then(|token_lit| Lit::from_token_lit(token_lit, token.span).ok())
226209
}
227210

228211
/// Attempts to recover an AST literal from semantic literal.

‎compiler/rustc_ast_lowering/src/expr.rs

+10-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use rustc_data_structures::stack::ensure_sufficient_stack;
1414
use rustc_hir as hir;
1515
use rustc_hir::def::Res;
1616
use rustc_hir::definitions::DefPathData;
17+
use rustc_session::errors::report_lit_error;
1718
use rustc_span::source_map::{respan, DesugaringKind, Span, Spanned};
1819
use rustc_span::symbol::{sym, Ident};
1920
use rustc_span::DUMMY_SP;
@@ -84,8 +85,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
8485
let ohs = self.lower_expr(ohs);
8586
hir::ExprKind::Unary(op, ohs)
8687
}
87-
ExprKind::Lit(ref l) => {
88-
hir::ExprKind::Lit(respan(self.lower_span(l.span), l.kind.clone()))
88+
ExprKind::Lit(token_lit) => {
89+
let lit_kind = match LitKind::from_token_lit(token_lit) {
90+
Ok(lit_kind) => lit_kind,
91+
Err(err) => {
92+
report_lit_error(&self.tcx.sess.parse_sess, err, token_lit, e.span);
93+
LitKind::Err
94+
}
95+
};
96+
hir::ExprKind::Lit(respan(self.lower_span(e.span), lit_kind))
8997
}
9098
ExprKind::IncludedBytes(ref bytes) => hir::ExprKind::Lit(respan(
9199
self.lower_span(e.span),

‎compiler/rustc_ast_lowering/src/lib.rs

+9-2
Original file line numberDiff line numberDiff line change
@@ -959,8 +959,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
959959
MacArgs::Eq(eq_span, MacArgsEq::Ast(ref expr)) => {
960960
// In valid code the value always ends up as a single literal. Otherwise, a dummy
961961
// literal suffices because the error is handled elsewhere.
962-
let lit = if let ExprKind::Lit(lit) = &expr.kind {
963-
lit.clone()
962+
let lit = if let ExprKind::Lit(token_lit) = expr.kind {
963+
match Lit::from_token_lit(token_lit, expr.span) {
964+
Ok(lit) => lit,
965+
Err(_err) => Lit {
966+
token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),
967+
kind: LitKind::Err,
968+
span: DUMMY_SP,
969+
},
970+
}
964971
} else {
965972
Lit {
966973
token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),

‎compiler/rustc_ast_pretty/src/pprust/state.rs

+7-3
Original file line numberDiff line numberDiff line change
@@ -373,8 +373,12 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
373373
}
374374

375375
fn print_literal(&mut self, lit: &ast::Lit) {
376-
self.maybe_print_comment(lit.span.lo());
377-
self.word(lit.token_lit.to_string())
376+
self.print_token_literal(lit.token_lit, lit.span)
377+
}
378+
379+
fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {
380+
self.maybe_print_comment(span.lo());
381+
self.word(token_lit.to_string())
378382
}
379383

380384
fn print_string(&mut self, st: &str, style: ast::StrStyle) {
@@ -1735,7 +1739,7 @@ impl<'a> State<'a> {
17351739
}
17361740
ast::Extern::Explicit(abi, _) => {
17371741
self.word_nbsp("extern");
1738-
self.print_literal(&abi.as_lit());
1742+
self.print_token_literal(abi.as_token_lit(), abi.span);
17391743
self.nbsp();
17401744
}
17411745
}

‎compiler/rustc_ast_pretty/src/pprust/state/expr.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -319,8 +319,8 @@ impl<'a> State<'a> {
319319
ast::ExprKind::AddrOf(k, m, ref expr) => {
320320
self.print_expr_addr_of(k, m, expr);
321321
}
322-
ast::ExprKind::Lit(ref lit) => {
323-
self.print_literal(lit);
322+
ast::ExprKind::Lit(token_lit) => {
323+
self.print_token_literal(token_lit, expr.span);
324324
}
325325
ast::ExprKind::IncludedBytes(ref bytes) => {
326326
let lit = ast::Lit::from_included_bytes(bytes, expr.span);

‎compiler/rustc_ast_pretty/src/pprust/state/item.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ impl<'a> State<'a> {
207207
s.word("extern");
208208
}));
209209
if let Some(abi) = nmod.abi {
210-
self.print_literal(&abi.as_lit());
210+
self.print_token_literal(abi.as_token_lit(), abi.span);
211211
self.nbsp();
212212
}
213213
self.bopen();

‎compiler/rustc_builtin_macros/src/asm.rs

+5-1
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,11 @@ pub fn parse_asm_args<'a>(
172172
// If it can't possibly expand to a string, provide diagnostics here to include other
173173
// things it could have been.
174174
match template.kind {
175-
ast::ExprKind::Lit(ast::Lit { kind: ast::LitKind::Str(..), .. }) => {}
175+
ast::ExprKind::Lit(token_lit)
176+
if matches!(
177+
token_lit.kind,
178+
token::LitKind::Str | token::LitKind::StrRaw(_)
179+
) => {}
176180
ast::ExprKind::MacCall(..) => {}
177181
_ => {
178182
let errstr = if is_global_asm {

‎compiler/rustc_builtin_macros/src/concat.rs

+13-12
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use rustc_ast as ast;
22
use rustc_ast::tokenstream::TokenStream;
33
use rustc_expand::base::{self, DummyResult};
4+
use rustc_session::errors::report_lit_error;
45
use rustc_span::symbol::Symbol;
56

67
use std::string::String;
@@ -18,28 +19,28 @@ pub fn expand_concat(
1819
let mut has_errors = false;
1920
for e in es {
2021
match e.kind {
21-
ast::ExprKind::Lit(ref lit) => match lit.kind {
22-
ast::LitKind::Str(ref s, _) | ast::LitKind::Float(ref s, _) => {
22+
ast::ExprKind::Lit(token_lit) => match ast::LitKind::from_token_lit(token_lit) {
23+
Ok(ast::LitKind::Str(ref s, _) | ast::LitKind::Float(ref s, _)) => {
2324
accumulator.push_str(s.as_str());
2425
}
25-
ast::LitKind::Char(c) => {
26+
Ok(ast::LitKind::Char(c)) => {
2627
accumulator.push(c);
2728
}
28-
ast::LitKind::Int(
29-
i,
30-
ast::LitIntType::Unsigned(_)
31-
| ast::LitIntType::Signed(_)
32-
| ast::LitIntType::Unsuffixed,
33-
) => {
29+
Ok(ast::LitKind::Int(i, _)) => {
3430
accumulator.push_str(&i.to_string());
3531
}
36-
ast::LitKind::Bool(b) => {
32+
Ok(ast::LitKind::Bool(b)) => {
3733
accumulator.push_str(&b.to_string());
3834
}
39-
ast::LitKind::Byte(..) | ast::LitKind::ByteStr(..) => {
35+
Ok(ast::LitKind::Byte(..) | ast::LitKind::ByteStr(..)) => {
4036
cx.span_err(e.span, "cannot concatenate a byte string literal");
37+
has_errors = true;
38+
}
39+
Ok(ast::LitKind::Err) => {
40+
has_errors = true;
4141
}
42-
ast::LitKind::Err => {
42+
Err(err) => {
43+
report_lit_error(&cx.sess.parse_sess, err, token_lit, e.span);
4344
has_errors = true;
4445
}
4546
},

0 commit comments

Comments
 (0)