Rollup merge of rust-lang#120221 - compiler-errors:statements-are-not…

…-patterns, r=nnethercote

Don't make statement nonterminals match pattern nonterminals

Right now, the heuristic we use to check if a token may begin a pattern nonterminal falls back to `may_be_ident`:
https://github.com/rust-lang/rust/blob/ef71f1047e04438181d7cb925a833e2ada6ab390/compiler/rustc_parse/src/parser/nonterminal.rs#L21-L37

This has the unfortunate side effect that a `stmt` nonterminal eagerly matches against a `pat` nonterminal, leading to a parse error:
```rust
macro_rules! m {
    ($pat:pat) => {};
    ($stmt:stmt) => {};
}

macro_rules! m2 {
    ($stmt:stmt) => {
        m! { $stmt }
    };
}

m2! { let x = 1 }
```

This PR fixes it by more accurately reflecting the set of nonterminals that may begin a pattern nonterminal.

As a side-effect, I modified `Token::can_begin_pattern` to work correctly and used that in `Parser::nonterminal_may_begin_with`.

compiler/rustc_ast/src/token.rs

@@ -486,6 +486,9 @@ impl Token {
  }
 
  /// Returns `true` if the token can appear at the start of an expression.
+ ///
+ /// **NB**: Take care when modifying this function, since it will change
+ /// the stable set of tokens that are allowed to match an expr nonterminal.
  pub fn can_begin_expr(&self) -> bool {
  match self.uninterpolate().kind {
  Ident(name, is_raw) =>
@@ -504,34 +507,46 @@ impl Token {
  PathSep | // global path
  Lifetime(..) | // labeled loop
  Pound => true, // expression attributes
- Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
- NtExpr(..) |
- NtBlock(..) |
- NtPath(..)),
+ Interpolated(ref nt) =>
+ matches!(&**nt,
+ NtBlock(..) |
+ NtExpr(..) |
+ NtLiteral(..) |
+ NtPath(..)
+ ),
  _ => false,
  }
  }
 
  /// Returns `true` if the token can appear at the start of a pattern.
  ///
  /// Shamelessly borrowed from `can_begin_expr`, only used for diagnostics right now.
- pub fn can_begin_pattern(&self) -> bool {
- match self.uninterpolate().kind {
- Ident(name, is_raw) =>
- ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
- | OpenDelim(Delimiter::Bracket | Delimiter::Parenthesis) // tuple or array
- | Literal(..) // literal
- | BinOp(Minus) // unary minus
- | BinOp(And) // reference
- | AndAnd // double reference
- // DotDotDot is no longer supported
- | DotDot | DotDotDot | DotDotEq // ranges
- | Lt | BinOp(Shl) // associated path
- | PathSep => true, // global path
- Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
- NtPat(..) |
- NtBlock(..) |
- NtPath(..)),
+ pub fn can_begin_pattern(&self, pat_kind: NtPatKind) -> bool {
+ match &self.uninterpolate().kind {
+ // box, ref, mut, and other identifiers (can stricten)
+ Ident(..) | NtIdent(..) |
+ OpenDelim(Delimiter::Parenthesis) | // tuple pattern
+ OpenDelim(Delimiter::Bracket) | // slice pattern
+ BinOp(And) | // reference
+ BinOp(Minus) | // negative literal
+ AndAnd | // double reference
+ Literal(_) | // literal
+ DotDot | // range pattern (future compat)
+ DotDotDot | // range pattern (future compat)
+ PathSep | // path
+ Lt | // path (UFCS constant)
+ BinOp(Shl) => true, // path (double UFCS)
+ // leading vert `|` or-pattern
+ BinOp(Or) => matches!(pat_kind, PatWithOr),
+ Interpolated(nt) =>
+ matches!(&**nt,
+ | NtExpr(..)
+ | NtLiteral(..)
+ | NtMeta(..)
+ | NtPat(..)
+ | NtPath(..)
+ | NtTy(..)
+ ),
  _ => false,
  }
  }

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -86,25 +86,7 @@ impl<'a> Parser<'a> {
  token::Interpolated(nt) => may_be_ident(nt),
  _ => false,
  },
- NonterminalKind::Pat(pat_kind) => match &token.kind {
- // box, ref, mut, and other identifiers (can stricten)
- token::Ident(..) | token::NtIdent(..) |
- token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern
- token::OpenDelim(Delimiter::Bracket) | // slice pattern
- token::BinOp(token::And) | // reference
- token::BinOp(token::Minus) | // negative literal
- token::AndAnd | // double reference
- token::Literal(_) | // literal
- token::DotDot | // range pattern (future compat)
- token::DotDotDot | // range pattern (future compat)
- token::PathSep | // path
- token::Lt | // path (UFCS constant)
- token::BinOp(token::Shl) => true, // path (double UFCS)
- // leading vert `|` or-pattern
- token::BinOp(token::Or) => matches!(pat_kind, PatWithOr),
- token::Interpolated(nt) => may_be_ident(nt),
- _ => false,
- },
+ NonterminalKind::Pat(pat_kind) => token.can_begin_pattern(pat_kind),
  NonterminalKind::Lifetime => match &token.kind {
  token::Lifetime(_) | token::NtLifetime(..) => true,
  _ => false,

compiler/rustc_parse/src/parser/pat.rs

@@ -444,7 +444,11 @@ impl<'a> Parser<'a> {
 
  let mut lo = self.token.span;
 
- if self.token.is_keyword(kw::Let) && self.look_ahead(1, |tok| tok.can_begin_pattern()) {
+ if self.token.is_keyword(kw::Let)
+ && self.look_ahead(1, |tok| {
+ tok.can_begin_pattern(token::NtPatKind::PatParam { inferred: false })
+ })
+ {
  self.bump();
  self.dcx().emit_err(RemoveLet { span: lo });
  lo = self.token.span;

compiler/rustc_parse/src/parser/path.rs

@@ -378,7 +378,10 @@ impl<'a> Parser<'a> {
  if self.may_recover()
  && prev_token_before_parsing == token::PathSep
  && (style == PathStyle::Expr && self.token.can_begin_expr()
- || style == PathStyle::Pat && self.token.can_begin_pattern())
+ || style == PathStyle::Pat
+ && self.token.can_begin_pattern(token::NtPatKind::PatParam {
+ inferred: false,
+ }))
  {
  snapshot = Some(self.create_snapshot_for_diagnostic());
  }

tests/ui/pattern/patterns-dont-match-nt-statement.rs

@@ -0,0 +1,19 @@
+//@ check-pass
+
+// Make sure that a `stmt` nonterminal does not eagerly match against
+// a `pat`, since this will always cause a parse error...
+
+macro_rules! m {
+ ($pat:pat) => {};
+ ($stmt:stmt) => {};
+}
+
+macro_rules! m2 {
+ ($stmt:stmt) => {
+ m! { $stmt }
+ };
+}
+
+m2! { let x = 1 }
+
+fn main() {}