Rollup merge of rust-lang#69801 - petrochenkov:nonorm, r=Centril

rustc_parse: Remove `Parser::normalized(_prev)_token`

Perform the "normalization" (renamed to "uninterpolation") on the fly when necessary.

The final part of rust-lang#69579 rust-lang#69384 rust-lang#69376 rust-lang#69211 rust-lang#69034 rust-lang#69006.
r? @Centril

src/librustc_ast/attr/mod.rs

@@ -286,6 +286,10 @@ impl MetaItem {
 }
 
 impl AttrItem {
+ pub fn span(&self) -> Span {
+ self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
+ }
+
  pub fn meta(&self, span: Span) -> Option<MetaItem> {
  Some(MetaItem {
  path: self.path.clone(),
@@ -437,7 +441,7 @@ impl MetaItem {
  I: Iterator<Item = TokenTree>,
  {
  // FIXME: Share code with `parse_path`.
- let path = match tokens.next() {
+ let path = match tokens.next().map(TokenTree::uninterpolate) {
  Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span }))
  | Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
  let mut segments = if let token::Ident(name, _) = kind {
@@ -453,7 +457,7 @@ impl MetaItem {
  };
  loop {
  if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
- tokens.next()
+ tokens.next().map(TokenTree::uninterpolate)
  {
  segments.push(PathSegment::from_ident(Ident::new(name, span)));
  } else {
@@ -470,7 +474,6 @@ impl MetaItem {
  Path { span, segments }
  }
  Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
- token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
  token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
  token::Nonterminal::NtPath(ref path) => path.clone(),
  _ => return None,

src/librustc_ast/token.rs

@@ -14,8 +14,8 @@ use rustc_macros::HashStable_Generic;
 use rustc_span::symbol::kw;
 use rustc_span::symbol::Symbol;
 use rustc_span::{self, Span, DUMMY_SP};
-use std::fmt;
-use std::mem;
+use std::borrow::Cow;
+use std::{fmt, mem};
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
 #[derive(HashStable_Generic)]
@@ -225,8 +225,15 @@ pub enum TokenKind {
  /* Literals */
  Literal(Lit),
 
- /* Name components */
+ /// Identifier token.
+ /// Do not forget about `NtIdent` when you want to match on identifiers.
+ /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
+ /// treat regular and interpolated identifiers in the same way.
  Ident(ast::Name, /* is_raw */ bool),
+ /// Lifetime identifier token.
+ /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
+ /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
+ /// treat regular and interpolated lifetime identifiers in the same way.
  Lifetime(ast::Name),
 
  Interpolated(Lrc<Nonterminal>),
@@ -328,6 +335,19 @@ impl Token {
  mem::replace(self, Token::dummy())
  }
 
+ /// For interpolated tokens, returns a span of the fragment to which the interpolated
+ /// token refers. For all other tokens this is just a regular span.
+ /// It is particularly important to use this for identifiers and lifetimes
+ /// for which spans affect name resolution and edition checks.
+ /// Note that keywords are also identifiers, so they should use this
+ /// if they keep spans or perform edition checks.
+ pub fn uninterpolated_span(&self) -> Span {
+ match &self.kind {
+ Interpolated(nt) => nt.span(),
+ _ => self.span,
+ }
+ }
+
  pub fn is_op(&self) -> bool {
  match self.kind {
  OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
@@ -345,7 +365,7 @@ impl Token {
 
  /// Returns `true` if the token can appear at the start of an expression.
  pub fn can_begin_expr(&self) -> bool {
- match self.kind {
+ match self.uninterpolate().kind {
  Ident(name, is_raw) =>
  ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
  OpenDelim(..) | // tuple, array or block
@@ -363,12 +383,10 @@ impl Token {
  Lifetime(..) | // labeled loop
  Pound => true, // expression attributes
  Interpolated(ref nt) => match **nt {
- NtIdent(ident, is_raw) => ident_can_begin_expr(ident.name, ident.span, is_raw),
  NtLiteral(..) |
  NtExpr(..) |
  NtBlock(..) |
- NtPath(..) |
- NtLifetime(..) => true,
+ NtPath(..) => true,
  _ => false,
  },
  _ => false,
@@ -377,7 +395,7 @@ impl Token {
 
  /// Returns `true` if the token can appear at the start of a type.
  pub fn can_begin_type(&self) -> bool {
- match self.kind {
+ match self.uninterpolate().kind {
  Ident(name, is_raw) =>
  ident_can_begin_type(name, self.span, is_raw), // type name or keyword
  OpenDelim(Paren) | // tuple
@@ -391,8 +409,7 @@ impl Token {
  Lt | BinOp(Shl) | // associated path
  ModSep => true, // global path
  Interpolated(ref nt) => match **nt {
- NtIdent(ident, is_raw) => ident_can_begin_type(ident.name, ident.span, is_raw),
- NtTy(..) | NtPath(..) | NtLifetime(..) => true,
+ NtTy(..) | NtPath(..) => true,
  _ => false,
  },
  _ => false,
@@ -433,38 +450,48 @@ impl Token {
  ///
  /// Keep this in sync with `Lit::from_token`.
  pub fn can_begin_literal_or_bool(&self) -> bool {
- match self.kind {
+ match self.uninterpolate().kind {
  Literal(..) | BinOp(Minus) => true,
  Ident(name, false) if name.is_bool_lit() => true,
  Interpolated(ref nt) => match &**nt {
- NtIdent(ident, false) if ident.name.is_bool_lit() => true,
  NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
  _ => false,
  },
  _ => false,
  }
  }
 
+ // A convenience function for matching on identifiers during parsing.
+ // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
+ // into the regular identifier or lifetime token it refers to,
+ // otherwise returns the original token.
+ pub fn uninterpolate(&self) -> Cow<'_, Token> {
+ match &self.kind {
+ Interpolated(nt) => match **nt {
+ NtIdent(ident, is_raw) => {
+ Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
+ }
+ NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
+ _ => Cow::Borrowed(self),
+ },
+ _ => Cow::Borrowed(self),
+ }
+ }
+
  /// Returns an identifier if this token is an identifier.
  pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
- match self.kind {
- Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)),
- Interpolated(ref nt) => match **nt {
- NtIdent(ident, is_raw) => Some((ident, is_raw)),
- _ => None,
- },
+ let token = self.uninterpolate();
+ match token.kind {
+ Ident(name, is_raw) => Some((ast::Ident::new(name, token.span), is_raw)),
  _ => None,
  }
  }
 
  /// Returns a lifetime identifier if this token is a lifetime.
  pub fn lifetime(&self) -> Option<ast::Ident> {
- match self.kind {
- Lifetime(name) => Some(ast::Ident::new(name, self.span)),
- Interpolated(ref nt) => match **nt {
- NtLifetime(ident) => Some(ident),
- _ => None,
- },
+ let token = self.uninterpolate();
+ match token.kind {
+ Lifetime(name) => Some(ast::Ident::new(name, token.span)),
  _ => None,
  }
  }
@@ -714,6 +741,24 @@ pub enum Nonterminal {
 #[cfg(target_arch = "x86_64")]
 rustc_data_structures::static_assert_size!(Nonterminal, 40);
 
+impl Nonterminal {
+ fn span(&self) -> Span {
+ match self {
+ NtItem(item) => item.span,
+ NtBlock(block) => block.span,
+ NtStmt(stmt) => stmt.span,
+ NtPat(pat) => pat.span,
+ NtExpr(expr) | NtLiteral(expr) => expr.span,
+ NtTy(ty) => ty.span,
+ NtIdent(ident, _) | NtLifetime(ident) => ident.span,
+ NtMeta(attr_item) => attr_item.span(),
+ NtPath(path) => path.span,
+ NtVis(vis) => vis.span,
+ NtTT(tt) => tt.span(),
+ }
+ }
+}
+
 impl PartialEq for Nonterminal {
  fn eq(&self, rhs: &Self) -> bool {
  match (self, rhs) {

src/librustc_ast/tokenstream.rs

@@ -116,6 +116,13 @@ impl TokenTree {
  pub fn close_tt(span: DelimSpan, delim: DelimToken) -> TokenTree {
  TokenTree::token(token::CloseDelim(delim), span.close)
  }
+
+ pub fn uninterpolate(self) -> TokenTree {
+ match self {
+ TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
+ tt => tt,
+ }
+ }
 }
 
 impl<CTX> HashStable<CTX> for TokenStream

src/librustc_ast/util/literal.rs

@@ -191,23 +191,16 @@ impl Lit {
  ///
  /// Keep this in sync with `Token::can_begin_literal_or_bool`.
  pub fn from_token(token: &Token) -> Result<Lit, LitError> {
- let lit = match token.kind {
+ let lit = match token.uninterpolate().kind {
  token::Ident(name, false) if name.is_bool_lit() => {
  token::Lit::new(token::Bool, name, None)
  }
  token::Literal(lit) => lit,
  token::Interpolated(ref nt) => {
- match &**nt {
- token::NtIdent(ident, false) if ident.name.is_bool_lit() => {
- let lit = token::Lit::new(token::Bool, ident.name, None);
- return Lit::from_lit_token(lit, ident.span);
+ if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
+ if let ast::ExprKind::Lit(lit) = &expr.kind {
+ return Ok(lit.clone());
  }
- token::NtExpr(expr) | token::NtLiteral(expr) => {
- if let ast::ExprKind::Lit(lit) = &expr.kind {
- return Ok(lit.clone());
- }
- }
- _ => {}
  }
  return Err(LitError::NotLiteral);
  }

src/librustc_builtin_macros/format.rs

@@ -156,44 +156,43 @@ fn parse_args<'a>(
  if p.token == token::Eof {
  break;
  } // accept trailing commas
- if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
- named = true;
- let name = if let token::Ident(name, _) = p.normalized_token.kind {
+ match p.token.ident() {
+ Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => {
+  named = true;
  p.bump();
- name
- } else {
- unreachable!();
- };
+ p.expect(&token::Eq)?;
+ let e = p.parse_expr()?;
+ if let Some(prev) = names.get(&ident.name) {
+ ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", ident))
+ .span_label(args[*prev].span, "previously here")
+ .span_label(e.span, "duplicate argument")
+ .emit();
+ continue;
+ }
 
- p.expect(&token::Eq)?;
- let e = p.parse_expr()?;
- if let Some(prev) = names.get(&name) {
- ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", name))
- .span_label(args[*prev].span, "previously here")
- .span_label(e.span, "duplicate argument")
- .emit();
- continue;
+ // Resolve names into slots early.
+ // Since all the positional args are already seen at this point
+ // if the input is valid, we can simply append to the positional
+ // args. And remember the names.
+ let slot = args.len();
+ names.insert(ident.name, slot);
+ args.push(e);
  }
-
- // Resolve names into slots early.
- // Since all the positional args are already seen at this point
- // if the input is valid, we can simply append to the positional
- // args. And remember the names.
- let slot = args.len();
- names.insert(name, slot);
- args.push(e);
- } else {
- let e = p.parse_expr()?;
- if named {
- let mut err = ecx
- .struct_span_err(e.span, "positional arguments cannot follow named arguments");
- err.span_label(e.span, "positional arguments must be before named arguments");
- for pos in names.values() {
- err.span_label(args[*pos].span, "named argument");
+ _ => {
+ let e = p.parse_expr()?;
+ if named {
+ let mut err = ecx.struct_span_err(
+ e.span,
+ "positional arguments cannot follow named arguments",
+ );
+ err.span_label(e.span, "positional arguments must be before named arguments");
+ for pos in names.values() {
+ err.span_label(args[*pos].span, "named argument");
+ }
+ err.emit();
  }
- err.emit();
+ args.push(e);
  }
- args.push(e);
  }
  }
  Ok((fmtstr, args, names))

src/librustc_expand/mbe/macro_parser.rs

@@ -750,17 +750,8 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
 
 /// The token is an identifier, but not `_`.
 /// We prohibit passing `_` to macros expecting `ident` for now.
-fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
- match token.kind {
- token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
- token::Interpolated(ref nt) => match **nt {
- token::NtIdent(ident, is_raw) if ident.name != kw::Underscore => {
- Some((ident.name, is_raw))
- }
- _ => None,
- },
- _ => None,
- }
+fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
+ token.ident().filter(|(ident, _)| ident.name != kw::Underscore)
 }
 
 /// Checks whether a non-terminal may begin with a particular token.
@@ -783,7 +774,7 @@ fn may_begin_with(token: &Token, name: Name) -> bool {
  && !token.is_keyword(kw::Let)
  }
  sym::ty => token.can_begin_type(),
- sym::ident => get_macro_name(token).is_some(),
+ sym::ident => get_macro_ident(token).is_some(),
  sym::literal => token.can_begin_literal_or_bool(),
  sym::vis => match token.kind {
  // The follow-set of :vis + "priv" keyword + interpolated
@@ -888,9 +879,9 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
  sym::ty => token::NtTy(p.parse_ty()?),
  // this could be handled like a token, since it is one
  sym::ident => {
- if let Some((name, is_raw)) = get_macro_name(&p.token) {
+ if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
  p.bump();
- token::NtIdent(Ident::new(name, p.normalized_prev_token.span), is_raw)
+ token::NtIdent(ident, is_raw)
  } else {
  let token_str = pprust::token_to_string(&p.token);
  let msg = &format!("expected ident, found {}", &token_str);

src/librustc_parse/lib.rs

@@ -4,7 +4,7 @@
 #![feature(crate_visibility_modifier)]
 
 use rustc_ast::ast;
-use rustc_ast::token::{self, Nonterminal, Token};
+use rustc_ast::token::{self, Nonterminal};
 use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::sync::Lrc;
@@ -171,8 +171,7 @@ fn maybe_source_file_to_parser(
  let mut parser = stream_to_parser(sess, stream, None);
  parser.unclosed_delims = unclosed_delims;
  if parser.token == token::Eof {
- let span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
- parser.set_token(Token::new(token::Eof, span));
+ parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
  }
 
  Ok(parser)