diff --git a/src/librustc_ast/ast.rs b/src/librustc_ast/ast.rs index 68960ba9fe920..303fcc96e0636 100644 --- a/src/librustc_ast/ast.rs +++ b/src/librustc_ast/ast.rs @@ -492,6 +492,7 @@ pub struct Block { /// Distinguishes between `unsafe { ... }` and `{ ... }`. pub rules: BlockCheckMode, pub span: Span, + pub tokens: Option, } #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] @@ -499,6 +500,7 @@ pub struct Pat { pub id: NodeId, pub kind: PatKind, pub span: Span, + pub tokens: Option, } impl Pat { @@ -534,7 +536,7 @@ impl Pat { _ => return None, }; - Some(P(Ty { kind, id: self.id, span: self.span })) + Some(P(Ty { kind, id: self.id, span: self.span, tokens: None })) } /// Walk top-down and call `it` in each place where a pattern occurs @@ -875,6 +877,7 @@ pub struct Stmt { pub id: NodeId, pub kind: StmtKind, pub span: Span, + pub tokens: Option, } impl Stmt { @@ -1011,11 +1014,12 @@ pub struct Expr { pub kind: ExprKind, pub span: Span, pub attrs: AttrVec, + pub tokens: Option, } // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -rustc_data_structures::static_assert_size!(Expr, 96); +rustc_data_structures::static_assert_size!(Expr, 104); impl Expr { /// Returns `true` if this expression would be valid somewhere that expects a value; @@ -1091,7 +1095,7 @@ impl Expr { _ => return None, }; - Some(P(Ty { kind, id: self.id, span: self.span })) + Some(P(Ty { kind, id: self.id, span: self.span, tokens: None })) } pub fn precedence(&self) -> ExprPrecedence { @@ -1829,6 +1833,7 @@ pub struct Ty { pub id: NodeId, pub kind: TyKind, pub span: Span, + pub tokens: Option, } #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] @@ -2009,13 +2014,14 @@ impl Param { /// Builds a `Param` object from `ExplicitSelf`. pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param { let span = eself.span.to(eself_ident.span); - let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span }); + let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None }); let param = |mutbl, ty| Param { attrs, pat: P(Pat { id: DUMMY_NODE_ID, kind: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None), span, + tokens: None, }), span, ty, @@ -2031,6 +2037,7 @@ impl Param { id: DUMMY_NODE_ID, kind: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl }), span, + tokens: None, }), ), } diff --git a/src/librustc_ast/mut_visit.rs b/src/librustc_ast/mut_visit.rs index aa2968b3cbe44..19bcf06fbb0c2 100644 --- a/src/librustc_ast/mut_visit.rs +++ b/src/librustc_ast/mut_visit.rs @@ -449,7 +449,7 @@ pub fn noop_visit_ty_constraint( } pub fn noop_visit_ty(ty: &mut P, vis: &mut T) { - let Ty { id, kind, span } = ty.deref_mut(); + let Ty { id, kind, span, tokens: _ } = ty.deref_mut(); vis.visit_id(id); match kind { TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err | TyKind::Never | TyKind::CVarArgs => {} @@ -867,7 +867,7 @@ pub fn noop_visit_mt(MutTy { ty, mutbl: _ }: &mut MutTy, vis: &mu } pub fn noop_visit_block(block: &mut P, vis: &mut T) { - let Block { id, stmts, rules: _, span } = block.deref_mut(); + let Block { id, stmts, rules: _, span, tokens: _ } = block.deref_mut(); vis.visit_id(id); stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt)); vis.visit_span(span); @@ -1050,7 +1050,7 @@ pub fn noop_flat_map_foreign_item( } pub fn noop_visit_pat(pat: &mut P, vis: &mut T) { - let Pat { id, kind, span } = pat.deref_mut(); + let Pat { id, kind, span, tokens: _ } = pat.deref_mut(); vis.visit_id(id); match kind { PatKind::Wild | PatKind::Rest => {} @@ -1092,7 +1092,10 @@ pub fn noop_visit_anon_const(AnonConst { id, value }: &mut AnonCo vis.visit_expr(value); } -pub fn noop_visit_expr(Expr { kind, id, span, attrs }: &mut Expr, vis: &mut T) { +pub fn noop_visit_expr( + Expr { kind, id, span, attrs, tokens: _ }: &mut Expr, + vis: &mut T, +) { match kind { ExprKind::Box(expr) => vis.visit_expr(expr), ExprKind::Array(exprs) => visit_exprs(exprs, vis), @@ -1254,12 +1257,15 @@ pub fn noop_filter_map_expr(mut e: P, vis: &mut T) -> Optio } pub fn noop_flat_map_stmt( - Stmt { kind, mut span, mut id }: Stmt, + Stmt { kind, mut span, mut id, tokens: _ }: Stmt, vis: &mut T, ) -> SmallVec<[Stmt; 1]> { vis.visit_id(&mut id); vis.visit_span(&mut span); - noop_flat_map_stmt_kind(kind, vis).into_iter().map(|kind| Stmt { id, kind, span }).collect() + noop_flat_map_stmt_kind(kind, vis) + .into_iter() + .map(|kind| Stmt { id, kind, span, tokens: None }) + .collect() } pub fn noop_flat_map_stmt_kind( diff --git a/src/librustc_ast/token.rs b/src/librustc_ast/token.rs index 3fc6444168e24..b2ed49b4ee0ac 100644 --- a/src/librustc_ast/token.rs +++ b/src/librustc_ast/token.rs @@ -728,7 +728,7 @@ impl PartialEq for Token { } } -#[derive(Clone, RustcEncodable, RustcDecodable)] +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] /// For interpolation during macro expansion. pub enum Nonterminal { NtItem(P), @@ -749,7 +749,7 @@ pub enum Nonterminal { // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -rustc_data_structures::static_assert_size!(Nonterminal, 40); +rustc_data_structures::static_assert_size!(Nonterminal, 48); impl Nonterminal { fn span(&self) -> Span { @@ -786,7 +786,7 @@ impl PartialEq for Nonterminal { } } -impl fmt::Debug for Nonterminal { +/*impl fmt::Debug for Nonterminal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { NtItem(..) => f.pad("NtItem(..)"), @@ -804,7 +804,7 @@ impl fmt::Debug for Nonterminal { NtLifetime(..) => f.pad("NtLifetime(..)"), } } -} +}*/ impl HashStable for Nonterminal where diff --git a/src/librustc_ast_lowering/lib.rs b/src/librustc_ast_lowering/lib.rs index 32b0f0db3589d..2140ecdb1c9e3 100644 --- a/src/librustc_ast_lowering/lib.rs +++ b/src/librustc_ast_lowering/lib.rs @@ -1065,6 +1065,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { id: node_id, kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()), span: constraint.span, + tokens: None, }, itctx, ); @@ -1129,6 +1130,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { kind: ExprKind::Path(qself.clone(), path.clone()), span: ty.span, attrs: AttrVec::new(), + tokens: None, }; let ct = self.with_new_scopes(|this| hir::AnonConst { diff --git a/src/librustc_builtin_macros/asm.rs b/src/librustc_builtin_macros/asm.rs index 9811a6621afa5..311cf6cf4e542 100644 --- a/src/librustc_builtin_macros/asm.rs +++ b/src/librustc_builtin_macros/asm.rs @@ -61,6 +61,7 @@ pub fn expand_asm<'cx>( kind: ast::ExprKind::InlineAsm(P(inline_asm)), span: cx.with_def_site_ctxt(sp), attrs: ast::AttrVec::new(), + tokens: None, })) } diff --git a/src/librustc_builtin_macros/concat_idents.rs b/src/librustc_builtin_macros/concat_idents.rs index b55e71b2518f4..bdb6ae6db3969 100644 --- a/src/librustc_builtin_macros/concat_idents.rs +++ b/src/librustc_builtin_macros/concat_idents.rs @@ -52,6 +52,7 @@ pub fn expand_concat_idents<'cx>( kind: ast::ExprKind::Path(None, ast::Path::from_ident(self.ident)), span: self.ident.span, attrs: ast::AttrVec::new(), + tokens: None, })) } @@ -60,6 +61,7 @@ pub fn expand_concat_idents<'cx>( id: ast::DUMMY_NODE_ID, kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)), span: self.ident.span, + tokens: None, })) } } diff --git a/src/librustc_builtin_macros/deriving/debug.rs b/src/librustc_builtin_macros/deriving/debug.rs index f47be3c3c1975..3066966edb68a 100644 --- a/src/librustc_builtin_macros/deriving/debug.rs +++ b/src/librustc_builtin_macros/deriving/debug.rs @@ -133,5 +133,5 @@ fn stmt_let_undescore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P) -> ast span: sp, attrs: ast::AttrVec::new(), }); - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp } + ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp, tokens: None } } diff --git a/src/librustc_builtin_macros/deriving/mod.rs b/src/librustc_builtin_macros/deriving/mod.rs index b5ad67abf6201..94c83e0bde8c9 100644 --- a/src/librustc_builtin_macros/deriving/mod.rs +++ b/src/librustc_builtin_macros/deriving/mod.rs @@ -74,6 +74,7 @@ fn call_intrinsic( id: ast::DUMMY_NODE_ID, rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated), span, + tokens: None, })) } diff --git a/src/librustc_expand/base.rs b/src/librustc_expand/base.rs index f5f2a5ed43f2c..77250f418b9a7 100644 --- a/src/librustc_expand/base.rs +++ b/src/librustc_expand/base.rs @@ -391,6 +391,7 @@ macro_rules! make_stmts_default { id: ast::DUMMY_NODE_ID, span: e.span, kind: ast::StmtKind::Expr(e), + tokens: None }] }) }; @@ -542,6 +543,7 @@ impl MacResult for MacEager { id: ast::DUMMY_NODE_ID, span: e.span, kind: PatKind::Lit(e), + tokens: None, })); } } @@ -582,12 +584,13 @@ impl DummyResult { kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, span: sp, attrs: ast::AttrVec::new(), + tokens: None, }) } /// A plain dummy pattern. pub fn raw_pat(sp: Span) -> ast::Pat { - ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp } + ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None } } /// A plain dummy type. @@ -596,6 +599,7 @@ impl DummyResult { id: ast::DUMMY_NODE_ID, kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, span: sp, + tokens: None, }) } } @@ -630,6 +634,7 @@ impl MacResult for DummyResult { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), span: self.span, + tokens: None }]) } diff --git a/src/librustc_expand/build.rs b/src/librustc_expand/build.rs index 48ceaf5fccd8e..c2cb460fe1a11 100644 --- a/src/librustc_expand/build.rs +++ b/src/librustc_expand/build.rs @@ -53,7 +53,7 @@ impl<'a> ExtCtxt<'a> { } pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { - P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind }) + P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind, tokens: None }) } pub fn ty_path(&self, path: ast::Path) -> P { @@ -69,7 +69,13 @@ impl<'a> ExtCtxt<'a> { pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { ast::AnonConst { id: ast::DUMMY_NODE_ID, - value: P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }), + value: P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind, + span, + attrs: AttrVec::new(), + tokens: None, + }), } } @@ -151,7 +157,12 @@ impl<'a> ExtCtxt<'a> { } pub fn stmt_expr(&self, expr: P) -> ast::Stmt { - ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr) } + ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + tokens: None, + } } pub fn stmt_let( @@ -175,7 +186,12 @@ impl<'a> ExtCtxt<'a> { span: sp, attrs: AttrVec::new(), }); - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp } + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Local(local), + span: sp, + tokens: None, + } } // Generates `let _: Type;`, which is usually used for type assertions. @@ -188,11 +204,16 @@ impl<'a> ExtCtxt<'a> { span, attrs: AttrVec::new(), }); - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span } + ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span, tokens: None } } pub fn stmt_item(&self, sp: Span, item: P) -> ast::Stmt { - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(item), span: sp } + ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Item(item), + span: sp, + tokens: None, + } } pub fn block_expr(&self, expr: P) -> P { @@ -202,15 +223,22 @@ impl<'a> ExtCtxt<'a> { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr), + tokens: None, }], ) } pub fn block(&self, span: Span, stmts: Vec) -> P { - P(ast::Block { stmts, id: ast::DUMMY_NODE_ID, rules: BlockCheckMode::Default, span }) + P(ast::Block { + stmts, + id: ast::DUMMY_NODE_ID, + rules: BlockCheckMode::Default, + span, + tokens: None, + }) } pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P { - P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }) + P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new(), tokens: None }) } pub fn expr_path(&self, path: ast::Path) -> P { @@ -396,7 +424,7 @@ impl<'a> ExtCtxt<'a> { } pub fn pat(&self, span: Span, kind: PatKind) -> P { - P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) + P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span, tokens: None }) } pub fn pat_wild(&self, span: Span) -> P { self.pat(span, PatKind::Wild) diff --git a/src/librustc_expand/expand.rs b/src/librustc_expand/expand.rs index 4d0548f3f868a..c7efec6ad85db 100644 --- a/src/librustc_expand/expand.rs +++ b/src/librustc_expand/expand.rs @@ -1397,10 +1397,10 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { } // The placeholder expander gives ids to statements, so we avoid folding the id here. - let ast::Stmt { id, kind, span } = stmt; + let ast::Stmt { id, kind, span, tokens: _ } = stmt; noop_flat_map_stmt_kind(kind, self) .into_iter() - .map(|kind| ast::Stmt { id, kind, span }) + .map(|kind| ast::Stmt { id, kind, span, tokens: None }) .collect() } diff --git a/src/librustc_expand/placeholders.rs b/src/librustc_expand/placeholders.rs index e1781f8636e58..1d310ef3e15b3 100644 --- a/src/librustc_expand/placeholders.rs +++ b/src/librustc_expand/placeholders.rs @@ -33,10 +33,13 @@ pub fn placeholder( span, attrs: ast::AttrVec::new(), kind: ast::ExprKind::MacCall(mac_placeholder()), + tokens: None, }) }; - let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span }); - let pat = || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span }); + let ty = + || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span, tokens: None }); + let pat = + || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span, tokens: None }); match kind { AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), @@ -83,13 +86,17 @@ pub fn placeholder( id, span, kind: ast::PatKind::MacCall(mac_placeholder()), + tokens: None, + })), + AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty { + id, + span, + kind: ast::TyKind::MacCall(mac_placeholder()), + tokens: None, })), - AstFragmentKind::Ty => { - AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::MacCall(mac_placeholder()) })) - } AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ast::AttrVec::new())); - ast::Stmt { id, span, kind: ast::StmtKind::MacCall(mac) } + ast::Stmt { id, span, kind: ast::StmtKind::MacCall(mac), tokens: None } }]), AstFragmentKind::Arms => AstFragment::Arms(smallvec![ast::Arm { attrs: Default::default(), diff --git a/src/librustc_interface/util.rs b/src/librustc_interface/util.rs index df05bd7c5117d..7f3ba10a40212 100644 --- a/src/librustc_interface/util.rs +++ b/src/librustc_interface/util.rs @@ -711,6 +711,7 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> { rules, id: resolver.next_node_id(), span: rustc_span::DUMMY_SP, + tokens: None, } } @@ -720,12 +721,14 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> { kind: ast::ExprKind::Block(P(b), None), span: rustc_span::DUMMY_SP, attrs: AttrVec::new(), + tokens: None, }); ast::Stmt { id: resolver.next_node_id(), kind: ast::StmtKind::Expr(expr), span: rustc_span::DUMMY_SP, + tokens: None, } } @@ -735,12 +738,14 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> { id: self.resolver.next_node_id(), span: rustc_span::DUMMY_SP, attrs: AttrVec::new(), + tokens: None, }); let loop_stmt = ast::Stmt { id: self.resolver.next_node_id(), span: rustc_span::DUMMY_SP, kind: ast::StmtKind::Expr(loop_expr), + tokens: None, }; if self.within_static_or_const { diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs index 884499ff2dd48..8570faf6474d9 100644 --- a/src/librustc_parse/lib.rs +++ b/src/librustc_parse/lib.rs @@ -321,7 +321,12 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) } Nonterminal::NtTT(ref tt) => Some(tt.clone().into()), - _ => None, + Nonterminal::NtLiteral(ref expr) | Nonterminal::NtExpr(ref expr) => expr.tokens.clone(), + Nonterminal::NtTy(ref ty) => ty.tokens.clone(), + Nonterminal::NtBlock(ref block) => block.tokens.clone(), + Nonterminal::NtPat(ref pat) => pat.tokens.clone(), + Nonterminal::NtStmt(ref stmt) => stmt.tokens.clone(), + Nonterminal::NtMeta(_) | Nonterminal::NtPath(_) | Nonterminal::NtVis(_) => None, }; // FIXME(#43081): Avoid this pretty-print + reparse hack @@ -361,6 +366,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke going with stringified version" ); } + info!("nt_to_tokenstream: no tokens found at {:?} for {:?}", span, nt); return tokens_for_real; } diff --git a/src/librustc_parse/parser/diagnostics.rs b/src/librustc_parse/parser/diagnostics.rs index 8e52bb1614757..0a36bf6d84e9d 100644 --- a/src/librustc_parse/parser/diagnostics.rs +++ b/src/librustc_parse/parser/diagnostics.rs @@ -28,8 +28,9 @@ pub(super) fn dummy_arg(ident: Ident) -> Param { id: ast::DUMMY_NODE_ID, kind: PatKind::Ident(BindingMode::ByValue(Mutability::Not), ident, None), span: ident.span, + tokens: None, }); - let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID }; + let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID, tokens: None }; Param { attrs: AttrVec::default(), id: ast::DUMMY_NODE_ID, @@ -106,7 +107,12 @@ impl RecoverQPath for Ty { Some(P(self.clone())) } fn recovered(qself: Option, path: ast::Path) -> Self { - Self { span: path.span, kind: TyKind::Path(qself, path), id: ast::DUMMY_NODE_ID } + Self { + span: path.span, + kind: TyKind::Path(qself, path), + id: ast::DUMMY_NODE_ID, + tokens: None, + } } } @@ -115,7 +121,12 @@ impl RecoverQPath for Pat { self.to_ty() } fn recovered(qself: Option, path: ast::Path) -> Self { - Self { span: path.span, kind: PatKind::Path(qself, path), id: ast::DUMMY_NODE_ID } + Self { + span: path.span, + kind: PatKind::Path(qself, path), + id: ast::DUMMY_NODE_ID, + tokens: None, + } } } @@ -129,6 +140,7 @@ impl RecoverQPath for Expr { kind: ExprKind::Path(qself, path), attrs: AttrVec::new(), id: ast::DUMMY_NODE_ID, + tokens: None, } } } @@ -1363,7 +1375,8 @@ impl<'a> Parser<'a> { .emit(); // Pretend the pattern is `_`, to avoid duplicate errors from AST validation. - let pat = P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID }); + let pat = + P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID, tokens: None }); Ok((pat, ty)) } diff --git a/src/librustc_parse/parser/expr.rs b/src/librustc_parse/parser/expr.rs index c65e99842c5dd..513f69b49eae6 100644 --- a/src/librustc_parse/parser/expr.rs +++ b/src/librustc_parse/parser/expr.rs @@ -9,6 +9,7 @@ use rustc_ast::ast::{AnonConst, BinOp, BinOpKind, FnDecl, FnRetTy, MacCall, Para use rustc_ast::ast::{Arm, Async, BlockCheckMode, Expr, ExprKind, Label, Movability, RangeLimits}; use rustc_ast::ptr::P; use rustc_ast::token::{self, Token, TokenKind}; +use rustc_ast::tokenstream::TokenStream; use rustc_ast::util::classify; use rustc_ast::util::literal::LitError; use rustc_ast::util::parser::{prec_let_scrutinee_needs_par, AssocOp, Fixity}; @@ -901,92 +902,96 @@ impl<'a> Parser<'a> { // attributes by giving them a empty "already-parsed" list. let attrs = AttrVec::new(); - // Note: when adding new syntax here, don't forget to adjust `TokenKind::can_begin_expr()`. - let lo = self.token.span; - if let token::Literal(_) = self.token.kind { - // This match arm is a special-case of the `_` match arm below and - // could be removed without changing functionality, but it's faster - // to have it here, especially for programs with large constants. - self.parse_lit_expr(attrs) - } else if self.check(&token::OpenDelim(token::Paren)) { - self.parse_tuple_parens_expr(attrs) - } else if self.check(&token::OpenDelim(token::Brace)) { - self.parse_block_expr(None, lo, BlockCheckMode::Default, attrs) - } else if self.check(&token::BinOp(token::Or)) || self.check(&token::OrOr) { - self.parse_closure_expr(attrs) - } else if self.check(&token::OpenDelim(token::Bracket)) { - self.parse_array_or_repeat_expr(attrs) - } else if self.eat_lt() { - let (qself, path) = self.parse_qpath(PathStyle::Expr)?; - Ok(self.mk_expr(lo.to(path.span), ExprKind::Path(Some(qself), path), attrs)) - } else if self.check_path() { - self.parse_path_start_expr(attrs) - } else if self.check_keyword(kw::Move) || self.check_keyword(kw::Static) { - self.parse_closure_expr(attrs) - } else if self.eat_keyword(kw::If) { - self.parse_if_expr(attrs) - } else if self.eat_keyword(kw::For) { - self.parse_for_expr(None, self.prev_token.span, attrs) - } else if self.eat_keyword(kw::While) { - self.parse_while_expr(None, self.prev_token.span, attrs) - } else if let Some(label) = self.eat_label() { - self.parse_labeled_expr(label, attrs) - } else if self.eat_keyword(kw::Loop) { - self.parse_loop_expr(None, self.prev_token.span, attrs) - } else if self.eat_keyword(kw::Continue) { - let kind = ExprKind::Continue(self.eat_label()); - Ok(self.mk_expr(lo.to(self.prev_token.span), kind, attrs)) - } else if self.eat_keyword(kw::Match) { - let match_sp = self.prev_token.span; - self.parse_match_expr(attrs).map_err(|mut err| { - err.span_label(match_sp, "while parsing this match expression"); - err - }) - } else if self.eat_keyword(kw::Unsafe) { - self.parse_block_expr(None, lo, BlockCheckMode::Unsafe(ast::UserProvided), attrs) - } else if self.is_do_catch_block() { - self.recover_do_catch(attrs) - } else if self.is_try_block() { - self.expect_keyword(kw::Try)?; - self.parse_try_block(lo, attrs) - } else if self.eat_keyword(kw::Return) { - self.parse_return_expr(attrs) - } else if self.eat_keyword(kw::Break) { - self.parse_break_expr(attrs) - } else if self.eat_keyword(kw::Yield) { - self.parse_yield_expr(attrs) - } else if self.eat_keyword(kw::Let) { - self.parse_let_expr(attrs) - } else if !self.unclosed_delims.is_empty() && self.check(&token::Semi) { - // Don't complain about bare semicolons after unclosed braces - // recovery in order to keep the error count down. Fixing the - // delimiters will possibly also fix the bare semicolon found in - // expression context. For example, silence the following error: - // - // error: expected expression, found `;` - // --> file.rs:2:13 - // | - // 2 | foo(bar(; - // | ^ expected expression - self.bump(); - Ok(self.mk_expr_err(self.token.span)) - } else if self.token.uninterpolated_span().rust_2018() { - // `Span::rust_2018()` is somewhat expensive; don't get it repeatedly. - if self.check_keyword(kw::Async) { - if self.is_async_block() { - // Check for `async {` and `async move {`. - self.parse_async_block(attrs) + let (mut expr, tokens) = self.collect_tokens(|this| { + // Note: when adding new syntax here, don't forget to adjust `TokenKind::can_begin_expr()`. + let lo = this.token.span; + if let token::Literal(_) = this.token.kind { + // This match arm is a special-case of the `_` match arm below and + // could be removed without changing functionality, but it's faster + // to have it here, especially for programs with large constants. + this.parse_lit_expr(attrs) + } else if this.check(&token::OpenDelim(token::Paren)) { + this.parse_tuple_parens_expr(attrs) + } else if this.check(&token::OpenDelim(token::Brace)) { + this.parse_block_expr(None, lo, BlockCheckMode::Default, attrs) + } else if this.check(&token::BinOp(token::Or)) || this.check(&token::OrOr) { + this.parse_closure_expr(attrs) + } else if this.check(&token::OpenDelim(token::Bracket)) { + this.parse_array_or_repeat_expr(attrs) + } else if this.eat_lt() { + let (qself, path) = this.parse_qpath(PathStyle::Expr)?; + Ok(this.mk_expr(lo.to(path.span), ExprKind::Path(Some(qself), path), attrs)) + } else if this.check_path() { + this.parse_path_start_expr(attrs) + } else if this.check_keyword(kw::Move) || this.check_keyword(kw::Static) { + this.parse_closure_expr(attrs) + } else if this.eat_keyword(kw::If) { + this.parse_if_expr(attrs) + } else if this.eat_keyword(kw::For) { + this.parse_for_expr(None, this.prev_token.span, attrs) + } else if this.eat_keyword(kw::While) { + this.parse_while_expr(None, this.prev_token.span, attrs) + } else if let Some(label) = this.eat_label() { + this.parse_labeled_expr(label, attrs) + } else if this.eat_keyword(kw::Loop) { + this.parse_loop_expr(None, this.prev_token.span, attrs) + } else if this.eat_keyword(kw::Continue) { + let kind = ExprKind::Continue(this.eat_label()); + Ok(this.mk_expr(lo.to(this.prev_token.span), kind, attrs)) + } else if this.eat_keyword(kw::Match) { + let match_sp = this.prev_token.span; + this.parse_match_expr(attrs).map_err(|mut err| { + err.span_label(match_sp, "while parsing this match expression"); + err + }) + } else if this.eat_keyword(kw::Unsafe) { + this.parse_block_expr(None, lo, BlockCheckMode::Unsafe(ast::UserProvided), attrs) + } else if this.is_do_catch_block() { + this.recover_do_catch(attrs) + } else if this.is_try_block() { + this.expect_keyword(kw::Try)?; + this.parse_try_block(lo, attrs) + } else if this.eat_keyword(kw::Return) { + this.parse_return_expr(attrs) + } else if this.eat_keyword(kw::Break) { + this.parse_break_expr(attrs) + } else if this.eat_keyword(kw::Yield) { + this.parse_yield_expr(attrs) + } else if this.eat_keyword(kw::Let) { + this.parse_let_expr(attrs) + } else if !this.unclosed_delims.is_empty() && this.check(&token::Semi) { + // Don't complain about bare semicolons after unclosed braces + // recovery in order to keep the error count down. Fixing the + // delimiters will possibly also fix the bare semicolon found in + // expression context. For example, silence the following error: + // + // error: expected expression, found `;` + // --> file.rs:2:13 + // | + // 2 | foo(bar(; + // | ^ expected expression + this.bump(); + Ok(this.mk_expr_err(this.token.span)) + } else if this.token.uninterpolated_span().rust_2018() { + // `Span::rust_2018()` is somewhat expensive; don't get it repeatedly. + if this.check_keyword(kw::Async) { + if this.is_async_block() { + // Check for `async {` and `async move {`. + this.parse_async_block(attrs) + } else { + this.parse_closure_expr(attrs) + } + } else if this.eat_keyword(kw::Await) { + this.recover_incorrect_await_syntax(lo, this.prev_token.span, attrs) } else { - self.parse_closure_expr(attrs) + this.parse_lit_expr(attrs) } - } else if self.eat_keyword(kw::Await) { - self.recover_incorrect_await_syntax(lo, self.prev_token.span, attrs) } else { - self.parse_lit_expr(attrs) + this.parse_lit_expr(attrs) } - } else { - self.parse_lit_expr(attrs) - } + })?; + expr.tokens = Some(tokens); + Ok(expr) } fn parse_lit_expr(&mut self, attrs: AttrVec) -> PResult<'a, P> { @@ -2121,8 +2126,18 @@ impl<'a> Parser<'a> { Ok(await_expr) } + crate fn mk_expr_with_tokens( + &self, + span: Span, + kind: ExprKind, + attrs: AttrVec, + tokens: Option, + ) -> P { + P(Expr { kind, span, attrs, id: DUMMY_NODE_ID, tokens }) + } + crate fn mk_expr(&self, span: Span, kind: ExprKind, attrs: AttrVec) -> P { - P(Expr { kind, span, attrs, id: DUMMY_NODE_ID }) + self.mk_expr_with_tokens(span, kind, attrs, None) } pub(super) fn mk_expr_err(&self, span: Span) -> P { diff --git a/src/librustc_parse/parser/item.rs b/src/librustc_parse/parser/item.rs index e927bcd07e2cd..f714deb343ebf 100644 --- a/src/librustc_parse/parser/item.rs +++ b/src/librustc_parse/parser/item.rs @@ -425,7 +425,12 @@ impl<'a> Parser<'a> { { let span = self.prev_token.span.between(self.token.span); self.struct_span_err(span, "missing trait in a trait impl").emit(); - P(Ty { kind: TyKind::Path(None, err_path(span)), span, id: DUMMY_NODE_ID }) + P(Ty { + kind: TyKind::Path(None, err_path(span)), + span, + id: DUMMY_NODE_ID, + tokens: None, + }) } else { self.parse_ty()? }; @@ -946,7 +951,7 @@ impl<'a> Parser<'a> { // The user intended that the type be inferred, // so treat this as if the user wrote e.g. `const A: _ = expr;`. - P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID }) + P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID, tokens: None }) } /// Parses an enum declaration. diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index 9376c7c1c724d..a60bcf01ec892 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -145,6 +145,7 @@ struct TokenCursorFrame { tree_cursor: tokenstream::Cursor, close_delim: bool, last_token: LastToken, + original_tree: Option, } /// This is used in `TokenCursorFrame` above to track tokens that are consumed @@ -171,7 +172,12 @@ enum LastToken { } impl TokenCursorFrame { - fn new(span: DelimSpan, delim: DelimToken, tts: &TokenStream) -> Self { + fn new( + span: DelimSpan, + delim: DelimToken, + tts: &TokenStream, + original_tree: Option, + ) -> Self { TokenCursorFrame { delim, span, @@ -179,6 +185,7 @@ impl TokenCursorFrame { tree_cursor: tts.clone().into_trees(), close_delim: delim == token::NoDelim, last_token: LastToken::Was(None), + original_tree, } } } @@ -206,10 +213,10 @@ impl TokenCursor { LastToken::Was(ref mut t) => *t = Some(tree.clone().into()), } - match tree { + match tree.clone() { TokenTree::Token(token) => return token, TokenTree::Delimited(sp, delim, tts) => { - let frame = TokenCursorFrame::new(sp, delim, &tts); + let frame = TokenCursorFrame::new(sp, delim, &tts, Some(tree)); self.stack.push(mem::replace(&mut self.frame, frame)); } } @@ -270,6 +277,7 @@ impl TokenCursor { .cloned() .collect::() }, + None, ), )); @@ -373,7 +381,7 @@ impl<'a> Parser<'a> { root_module_name: None, expected_tokens: Vec::new(), token_cursor: TokenCursor { - frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens), + frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens, None), stack: Vec::new(), }, desugar_doc_comments, @@ -1147,6 +1155,13 @@ impl<'a> Parser<'a> { &mut self, f: impl FnOnce(&mut Self) -> PResult<'a, R>, ) -> PResult<'a, (R, TokenStream)> { + if matches!(self.token.kind, TokenKind::OpenDelim(_)) { + let tokens = vec![ + self.token_cursor.frame.original_tree.clone().expect("Missing token tree!").into(), + ]; + let ret = f(self); + return Ok((ret?, TokenStream::new(tokens))); + } // Record all tokens we parse when parsing this item. let mut tokens = Vec::new(); let prev_collecting = match self.token_cursor.frame.last_token { diff --git a/src/librustc_parse/parser/pat.rs b/src/librustc_parse/parser/pat.rs index 4585941943b74..ce9c364d53344 100644 --- a/src/librustc_parse/parser/pat.rs +++ b/src/librustc_parse/parser/pat.rs @@ -38,7 +38,10 @@ impl<'a> Parser<'a> { /// at the top level. Used when parsing the parameters of lambda expressions, /// functions, function pointers, and `pat` macro fragments. pub fn parse_pat(&mut self, expected: Expected) -> PResult<'a, P> { - self.parse_pat_with_range_pat(true, expected) + let (mut pat, tokens) = + self.collect_tokens(|this| this.parse_pat_with_range_pat(true, expected))?; + pat.tokens = Some(tokens); + Ok(pat) } /// Entry point to the main pattern parser. @@ -986,6 +989,6 @@ impl<'a> Parser<'a> { } fn mk_pat(&self, span: Span, kind: PatKind) -> P { - P(Pat { kind, span, id: ast::DUMMY_NODE_ID }) + P(Pat { kind, span, id: ast::DUMMY_NODE_ID, tokens: None }) } } diff --git a/src/librustc_parse/parser/stmt.rs b/src/librustc_parse/parser/stmt.rs index 4359823be0890..d3bb362e030a8 100644 --- a/src/librustc_parse/parser/stmt.rs +++ b/src/librustc_parse/parser/stmt.rs @@ -17,17 +17,24 @@ use rustc_errors::{Applicability, PResult}; use rustc_span::source_map::{BytePos, Span}; use rustc_span::symbol::{kw, sym}; +use rustc_ast::tokenstream::TokenStream; use std::mem; impl<'a> Parser<'a> { /// Parses a statement. This stops just before trailing semicolons on everything but items. /// e.g., a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed. pub fn parse_stmt(&mut self) -> PResult<'a, Option> { - Ok(self.parse_stmt_without_recovery().unwrap_or_else(|mut e| { + let res = self.collect_tokens(|this| this.parse_stmt_without_recovery()); + let (mut stmt, tokens) = res.unwrap_or_else(|mut e| { e.emit(); self.recover_stmt_(SemiColonMode::Break, BlockMode::Ignore); - None - })) + (None, TokenStream::default()) + }); + + if let Some(stmt) = stmt.as_mut() { + stmt.tokens = Some(tokens) + } + Ok(stmt) } fn parse_stmt_without_recovery(&mut self) -> PResult<'a, Option> { @@ -236,10 +243,12 @@ impl<'a> Parser<'a> { /// Parses a block. No inner attributes are allowed. pub fn parse_block(&mut self) -> PResult<'a, P> { - let (attrs, block) = self.parse_inner_attrs_and_block()?; + let ((attrs, mut block), tokens) = + self.collect_tokens(|this| this.parse_inner_attrs_and_block())?; if let [.., last] = &*attrs { self.error_on_forbidden_inner_attr(last.span, DEFAULT_INNER_ATTR_FORBIDDEN); } + block.tokens = Some(tokens); Ok(block) } @@ -398,11 +407,11 @@ impl<'a> Parser<'a> { } pub(super) fn mk_block(&self, stmts: Vec, rules: BlockCheckMode, span: Span) -> P { - P(Block { stmts, id: DUMMY_NODE_ID, rules, span }) + P(Block { stmts, id: DUMMY_NODE_ID, rules, span, tokens: None }) } pub(super) fn mk_stmt(&self, span: Span, kind: StmtKind) -> Stmt { - Stmt { id: DUMMY_NODE_ID, kind, span } + Stmt { id: DUMMY_NODE_ID, kind, span, tokens: None } } fn mk_stmt_err(&self, span: Span) -> Stmt { diff --git a/src/librustc_parse/parser/ty.rs b/src/librustc_parse/parser/ty.rs index c21ac8d04f194..a44c3bfd801c4 100644 --- a/src/librustc_parse/parser/ty.rs +++ b/src/librustc_parse/parser/ty.rs @@ -62,7 +62,11 @@ fn can_continue_type_after_non_fn_ident(t: &Token) -> bool { impl<'a> Parser<'a> { /// Parses a type. pub fn parse_ty(&mut self) -> PResult<'a, P> { - self.parse_ty_common(AllowPlus::Yes, RecoverQPath::Yes, AllowCVariadic::No) + let (mut ty, tokens) = self.collect_tokens(|this| { + this.parse_ty_common(AllowPlus::Yes, RecoverQPath::Yes, AllowCVariadic::No) + })?; + ty.tokens = Some(tokens); + Ok(ty) } /// Parse a type suitable for a function or function pointer parameter. @@ -614,6 +618,6 @@ impl<'a> Parser<'a> { } pub(super) fn mk_ty(&self, span: Span, kind: TyKind) -> P { - P(Ty { kind, span, id: ast::DUMMY_NODE_ID }) + P(Ty { kind, span, id: ast::DUMMY_NODE_ID, tokens: None }) } } diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs index 365ae301c0fb5..4286e04cf8f35 100644 --- a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs +++ b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs @@ -55,6 +55,7 @@ fn expr(kind: ExprKind) -> P { kind, span: DUMMY_SP, attrs: ThinVec::new(), + tokens: None }) } @@ -111,6 +112,7 @@ fn iter_exprs(depth: usize, f: &mut dyn FnMut(P)) { id: DUMMY_NODE_ID, rules: BlockCheckMode::Default, span: DUMMY_SP, + tokens: None }); iter_exprs(depth - 1, &mut |e| g(ExprKind::If(e, block.clone(), None))); }, @@ -162,6 +164,7 @@ fn iter_exprs(depth: usize, f: &mut dyn FnMut(P)) { id: DUMMY_NODE_ID, kind: PatKind::Wild, span: DUMMY_SP, + tokens: None }); iter_exprs(depth - 1, &mut |e| g(ExprKind::Let(pat.clone(), e))) }, @@ -199,6 +202,7 @@ impl MutVisitor for AddParens { kind: ExprKind::Paren(e), span: DUMMY_SP, attrs: ThinVec::new(), + tokens: None }) }); }