Rollup merge of rust-lang#96433 - petrochenkov:delim, r=nnethercote

rustc_ast: Harmonize delimiter naming with `proc_macro::Delimiter`

Compiler cannot reuse `proc_macro::Delimiter` directly due to extra impls, but can at least use the same naming.

After this PR the only difference between these two enums is that `proc_macro::Delimiter::None` is turned into `token::Delimiter::Invisible`.
It's my mistake that the invisible delimiter is called `None` on stable, during the stabilization I audited the naming and wrote the docs, but missed the fact that the `None` naming gives a wrong and confusing impression about what this thing is.

cc rust-lang#96421
r? `@nnethercote`

Author: Dylan-DPC

compiler/rustc_ast/src/ast.rs

@@ -23,7 +23,7 @@ pub use GenericArgs::*;
 pub use UnsafeSource::*;
 
 use crate::ptr::P;
-use crate::token::{self, CommentKind, DelimToken, Token};
+use crate::token::{self, CommentKind, Delimiter, Token};
 use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree};
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -1542,7 +1542,7 @@ pub enum MacArgs {
 }
 
 impl MacArgs {
-    pub fn delim(&self) -> Option<DelimToken> {
+    pub fn delim(&self) -> Option<Delimiter> {
         match self {
             MacArgs::Delimited(_, delim, _) => Some(delim.to_token()),
             MacArgs::Empty | MacArgs::Eq(..) => None,
@@ -1582,20 +1582,20 @@ pub enum MacDelimiter {
 }
 
 impl MacDelimiter {
-    pub fn to_token(self) -> DelimToken {
+    pub fn to_token(self) -> Delimiter {
         match self {
-            MacDelimiter::Parenthesis => DelimToken::Paren,
-            MacDelimiter::Bracket => DelimToken::Bracket,
-            MacDelimiter::Brace => DelimToken::Brace,
+            MacDelimiter::Parenthesis => Delimiter::Parenthesis,
+            MacDelimiter::Bracket => Delimiter::Bracket,
+            MacDelimiter::Brace => Delimiter::Brace,
         }
     }
 
-    pub fn from_token(delim: DelimToken) -> Option<MacDelimiter> {
+    pub fn from_token(delim: Delimiter) -> Option<MacDelimiter> {
         match delim {
-            token::Paren => Some(MacDelimiter::Parenthesis),
-            token::Bracket => Some(MacDelimiter::Bracket),
-            token::Brace => Some(MacDelimiter::Brace),
-            token::NoDelim => None,
+            Delimiter::Parenthesis => Some(MacDelimiter::Parenthesis),
+            Delimiter::Bracket => Some(MacDelimiter::Bracket),
+            Delimiter::Brace => Some(MacDelimiter::Brace),
+            Delimiter::Invisible => None,
         }
     }
 }

compiler/rustc_ast/src/attr/mod.rs

@@ -5,7 +5,7 @@ use crate::ast::{AttrId, AttrItem, AttrKind, AttrStyle, Attribute};
 use crate::ast::{Lit, LitKind};
 use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
 use crate::ast::{Path, PathSegment};
-use crate::token::{self, CommentKind, Token};
+use crate::token::{self, CommentKind, Delimiter, Token};
 use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
 use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing};
 use crate::tokenstream::{LazyTokenStream, TokenStream};
@@ -513,7 +513,7 @@ impl MetaItemKind {
                 vec![
                     TokenTree::Delimited(
                         DelimSpan::from_single(span),
-                        token::Paren,
+                        Delimiter::Parenthesis,
                         TokenStream::new(tokens),
                     )
                     .into(),
@@ -540,7 +540,7 @@ impl MetaItemKind {
         tokens: &mut impl Iterator<Item = TokenTree>,
     ) -> Option<MetaItemKind> {
         match tokens.next() {
-            Some(TokenTree::Delimited(_, token::NoDelim, inner_tokens)) => {
+            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
             }
             Some(TokenTree::Token(token)) => {
@@ -565,7 +565,7 @@ impl MetaItemKind {
         tokens: &mut iter::Peekable<impl Iterator<Item = TokenTree>>,
     ) -> Option<MetaItemKind> {
         match tokens.peek() {
-            Some(TokenTree::Delimited(_, token::Paren, inner_tokens)) => {
+            Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
                 let inner_tokens = inner_tokens.clone();
                 tokens.next();
                 MetaItemKind::list_from_tokens(inner_tokens)
@@ -606,7 +606,7 @@ impl NestedMetaItem {
                 tokens.next();
                 return Some(NestedMetaItem::Literal(lit));
             }
-            Some(TokenTree::Delimited(_, token::NoDelim, inner_tokens)) => {
+            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
                 let inner_tokens = inner_tokens.clone();
                 tokens.next();
                 return NestedMetaItem::from_tokens(&mut inner_tokens.into_trees().peekable());

compiler/rustc_ast/src/token.rs

@@ -1,5 +1,4 @@
 pub use BinOpToken::*;
-pub use DelimToken::*;
 pub use LitKind::*;
 pub use Nonterminal::*;
 pub use TokenKind::*;
@@ -37,18 +36,26 @@ pub enum BinOpToken {
     Shr,
 }
 
-/// A delimiter token.
-#[derive(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Debug, Copy)]
-#[derive(HashStable_Generic)]
-pub enum DelimToken {
-    /// A round parenthesis (i.e., `(` or `)`).
-    Paren,
-    /// A square bracket (i.e., `[` or `]`).
-    Bracket,
-    /// A curly brace (i.e., `{` or `}`).
+/// Describes how a sequence of token trees is delimited.
+/// Cannot use `proc_macro::Delimiter` directly because this
+/// structure should implement some additional traits.
+/// The `None` variant is also renamed to `Invisible` to be
+/// less confusing and better convey the semantics.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Encodable, Decodable, Hash, HashStable_Generic)]
+pub enum Delimiter {
+    /// `( ... )`
+    Parenthesis,
+    /// `{ ... }`
     Brace,
-    /// An empty delimiter.
-    NoDelim,
+    /// `[ ... ]`
+    Bracket,
+    /// `Ø ... Ø`
+    /// An invisible delimiter, that may, for example, appear around tokens coming from a
+    /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
+    /// `$var * 3` where `$var` is `1 + 2`.
+    /// Invisible delimiters might not survive roundtrip of a token stream through a string.
+    Invisible,
 }
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@@ -212,9 +219,9 @@ pub enum TokenKind {
     /// Used by proc macros for representing lifetimes, not generated by lexer right now.
     SingleQuote,
     /// An opening delimiter (e.g., `{`).
-    OpenDelim(DelimToken),
+    OpenDelim(Delimiter),
     /// A closing delimiter (e.g., `}`).
-    CloseDelim(DelimToken),
+    CloseDelim(Delimiter),
 
     /* Literals */
     Literal(Lit),
@@ -387,8 +394,8 @@ impl Token {
         match self.uninterpolate().kind {
             Ident(name, is_raw)        =>
                 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
-            OpenDelim(Paren)            | // tuple
-            OpenDelim(Bracket)          | // array
+            OpenDelim(Delimiter::Parenthesis) | // tuple
+            OpenDelim(Delimiter::Bracket)     | // array
             Not                         | // never
             BinOp(Star)                 | // raw pointer
             BinOp(And)                  | // reference
@@ -405,7 +412,7 @@ impl Token {
     /// Returns `true` if the token can appear at the start of a const param.
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
-            OpenDelim(Brace) => true,
+            OpenDelim(Delimiter::Brace) => true,
             Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
             _ => self.can_begin_literal_maybe_minus(),
         }
@@ -417,7 +424,7 @@ impl Token {
             || self.is_lifetime()
             || self.is_keyword(kw::For)
             || self == &Question
-            || self == &OpenDelim(Paren)
+            || self == &OpenDelim(Delimiter::Parenthesis)
     }
 
     /// Returns `true` if the token is any literal.

compiler/rustc_ast/src/tokenstream.rs

@@ -13,7 +13,7 @@
 //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
 //! ownership of the original.
 
-use crate::token::{self, DelimToken, Token, TokenKind};
+use crate::token::{self, Delimiter, Token, TokenKind};
 use crate::AttrVec;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -42,7 +42,7 @@ pub enum TokenTree {
     /// A single token.
     Token(Token),
     /// A delimited sequence of token trees.
-    Delimited(DelimSpan, DelimToken, TokenStream),
+    Delimited(DelimSpan, Delimiter, TokenStream),
 }
 
 #[derive(Copy, Clone)]
@@ -57,7 +57,7 @@ fn _dummy()
 where
     Token: Send + Sync,
     DelimSpan: Send + Sync,
-    DelimToken: Send + Sync,
+    Delimiter: Send + Sync,
     TokenStream: Send + Sync,
 {
 }
@@ -175,7 +175,7 @@ pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing
 #[derive(Clone, Debug, Encodable, Decodable)]
 pub enum AttrAnnotatedTokenTree {
     Token(Token),
-    Delimited(DelimSpan, DelimToken, AttrAnnotatedTokenStream),
+    Delimited(DelimSpan, Delimiter, AttrAnnotatedTokenStream),
     /// Stores the attributes for an attribute target,
     /// along with the tokens for that attribute target.
     /// See `AttributesData` for more information

compiler/rustc_ast_lowering/src/lib.rs

@@ -38,7 +38,7 @@
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
 
-use rustc_ast::token::{self, Token};
+use rustc_ast::token::{Delimiter, Token};
 use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream, TokenTree};
 use rustc_ast::visit;
 use rustc_ast::{self as ast, *};
@@ -886,7 +886,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     match tokens.into_trees().next() {
                         Some(TokenTree::Token(token)) => token,
                         Some(TokenTree::Delimited(_, delim, tokens)) => {
-                            if delim != token::NoDelim {
+                            if delim != Delimiter::Invisible {
                                 sess.diagnostic().delay_span_bug(
                                     span,
                                     "unexpected delimiter in key-value attribute's value",

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -6,7 +6,7 @@ use crate::pp::Breaks::{Consistent, Inconsistent};
 use crate::pp::{self, Breaks};
 
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, BinOpToken, CommentKind, DelimToken, Nonterminal, Token, TokenKind};
+use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, Token, TokenKind};
 use rustc_ast::tokenstream::{TokenStream, TokenTree};
 use rustc_ast::util::classify;
 use rustc_ast::util::comments::{gather_comments, Comment, CommentStyle};
@@ -155,10 +155,10 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
     }
     match tt {
         TokenTree::Token(token) => !matches!(token.kind, token::Comma | token::Not | token::Dot),
-        TokenTree::Delimited(_, DelimToken::Paren, _) => {
+        TokenTree::Delimited(_, Delimiter::Parenthesis, _) => {
             !matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. }))
         }
-        TokenTree::Delimited(_, DelimToken::Bracket, _) => {
+        TokenTree::Delimited(_, Delimiter::Bracket, _) => {
             !matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. }))
         }
         TokenTree::Delimited(..) => true,
@@ -556,12 +556,12 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         header: Option<MacHeader<'_>>,
         has_bang: bool,
         ident: Option<Ident>,
-        delim: Option<DelimToken>,
+        delim: Option<Delimiter>,
         tts: &TokenStream,
         convert_dollar_crate: bool,
         span: Span,
     ) {
-        if delim == Some(DelimToken::Brace) {
+        if delim == Some(Delimiter::Brace) {
             self.cbox(INDENT_UNIT);
         }
         match header {
@@ -577,7 +577,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             self.print_ident(ident);
         }
         match delim {
-            Some(DelimToken::Brace) => {
+            Some(Delimiter::Brace) => {
                 if header.is_some() || has_bang || ident.is_some() {
                     self.nbsp();
                 }
@@ -758,13 +758,15 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::RArrow => "->".into(),
             token::LArrow => "<-".into(),
             token::FatArrow => "=>".into(),
-            token::OpenDelim(token::Paren) => "(".into(),
-            token::CloseDelim(token::Paren) => ")".into(),
-            token::OpenDelim(token::Bracket) => "[".into(),
-            token::CloseDelim(token::Bracket) => "]".into(),
-            token::OpenDelim(token::Brace) => "{".into(),
-            token::CloseDelim(token::Brace) => "}".into(),
-            token::OpenDelim(token::NoDelim) | token::CloseDelim(token::NoDelim) => "".into(),
+            token::OpenDelim(Delimiter::Parenthesis) => "(".into(),
+            token::CloseDelim(Delimiter::Parenthesis) => ")".into(),
+            token::OpenDelim(Delimiter::Bracket) => "[".into(),
+            token::CloseDelim(Delimiter::Bracket) => "]".into(),
+            token::OpenDelim(Delimiter::Brace) => "{".into(),
+            token::CloseDelim(Delimiter::Brace) => "}".into(),
+            token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
+                "".into()
+            }
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),

compiler/rustc_builtin_macros/src/asm.rs

@@ -1,6 +1,6 @@
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
-use rustc_ast::token;
+use rustc_ast::token::{self, Delimiter};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::{Applicability, PResult};
@@ -395,9 +395,9 @@ fn parse_options<'a>(
 ) -> PResult<'a, ()> {
     let span_start = p.prev_token.span;
 
-    p.expect(&token::OpenDelim(token::DelimToken::Paren))?;
+    p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
 
-    while !p.eat(&token::CloseDelim(token::DelimToken::Paren)) {
+    while !p.eat(&token::CloseDelim(Delimiter::Parenthesis)) {
         if !is_global_asm && p.eat_keyword(sym::pure) {
             try_set_option(p, args, sym::pure, ast::InlineAsmOptions::PURE);
         } else if !is_global_asm && p.eat_keyword(sym::nomem) {
@@ -421,7 +421,7 @@ fn parse_options<'a>(
         }
 
         // Allow trailing commas
-        if p.eat(&token::CloseDelim(token::DelimToken::Paren)) {
+        if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) {
             break;
         }
         p.expect(&token::Comma)?;
@@ -436,9 +436,9 @@ fn parse_options<'a>(
 fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a, ()> {
     let span_start = p.prev_token.span;
 
-    p.expect(&token::OpenDelim(token::DelimToken::Paren))?;
+    p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
 
-    if p.eat(&token::CloseDelim(token::DelimToken::Paren)) {
+    if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) {
         let err = p.sess.span_diagnostic.struct_span_err(
             p.token.span,
             "at least one abi must be provided as an argument to `clobber_abi`",
@@ -454,7 +454,7 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a,
             }
             Err(opt_lit) => {
                 // If the non-string literal is a closing paren then it's the end of the list and is fine
-                if p.eat(&token::CloseDelim(token::DelimToken::Paren)) {
+                if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) {
                     break;
                 }
                 let span = opt_lit.map_or(p.token.span, |lit| lit.span);
@@ -466,7 +466,7 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a,
         };
 
         // Allow trailing commas
-        if p.eat(&token::CloseDelim(token::DelimToken::Paren)) {
+        if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) {
             break;
         }
         p.expect(&token::Comma)?;
@@ -501,7 +501,7 @@ fn parse_reg<'a>(
     p: &mut Parser<'a>,
     explicit_reg: &mut bool,
 ) -> PResult<'a, ast::InlineAsmRegOrRegClass> {
-    p.expect(&token::OpenDelim(token::DelimToken::Paren))?;
+    p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
     let result = match p.token.uninterpolate().kind {
         token::Ident(name, false) => ast::InlineAsmRegOrRegClass::RegClass(name),
         token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
@@ -515,7 +515,7 @@ fn parse_reg<'a>(
         }
     };
     p.bump();
-    p.expect(&token::CloseDelim(token::DelimToken::Paren))?;
+    p.expect(&token::CloseDelim(Delimiter::Parenthesis))?;
     Ok(result)
 }
 

compiler/rustc_expand/src/config.rs

@@ -1,7 +1,7 @@
 //! Conditional compilation stripping.
 
 use rustc_ast::ptr::P;
-use rustc_ast::token::{DelimToken, Token, TokenKind};
+use rustc_ast::token::{Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
 use rustc_ast::tokenstream::{DelimSpan, Spacing};
 use rustc_ast::tokenstream::{LazyTokenStream, TokenTree};
@@ -418,7 +418,7 @@ impl<'a> StripUnconfigured<'a> {
         // in `#[attr]`, so just use the span of the `#` token.
         let bracket_group = AttrAnnotatedTokenTree::Delimited(
             DelimSpan::from_single(pound_span),
-            DelimToken::Bracket,
+            Delimiter::Bracket,
             item.tokens
                 .as_ref()
                 .unwrap_or_else(|| panic!("Missing tokens for {:?}", item))