Auto merge of #56369 - nnethercote:rm-Delimited, r=petrochenkov

Remove `tokenstream::Delimited`.

Because it's an extra type layer that doesn't really help; in a couple
of places it actively gets in the way, and overall removing it makes the
code nicer. It does, however, move `tokenstream::TokenTree` further away
from the `TokenTree` in `quote.rs`.

More importantly, this change reduces the size of `TokenStream` from 48
bytes to 40 bytes on x86-64, which is enough to slightly reduce
instruction counts on numerous benchmarks, the best by 1.5%.

Note that `open_tt` and `close_tt` have gone from being methods on
`Delimited` to associated methods of `TokenTree`.

src/librustc/hir/lowering.rs

@@ -72,7 +72,7 @@ use syntax::ptr::P;
 use syntax::source_map::{self, respan, CompilerDesugaringKind, Spanned};
 use syntax::std_inject;
 use syntax::symbol::{keywords, Symbol};
-use syntax::tokenstream::{Delimited, TokenStream, TokenTree};
+use syntax::tokenstream::{TokenStream, TokenTree};
 use syntax::parse::token::Token;
 use syntax::visit::{self, Visitor};
 use syntax_pos::{Span, MultiSpan};
@@ -1088,12 +1088,10 @@ impl<'a> LoweringContext<'a> {
     fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
         match tree {
             TokenTree::Token(span, token) => self.lower_token(token, span),
-            TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
+            TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
                 span,
-                Delimited {
-                    delim: delimited.delim,
-                    tts: self.lower_token_stream(delimited.tts.into()).into(),
-                },
+                delim,
+                self.lower_token_stream(tts.into()).into(),
             ).into(),
         }
     }

src/librustc/ich/impls_syntax.rs

@@ -265,10 +265,10 @@ for tokenstream::TokenTree {
                 span.hash_stable(hcx, hasher);
                 hash_token(token, hcx, hasher);
             }
-            tokenstream::TokenTree::Delimited(span, ref delimited) => {
+            tokenstream::TokenTree::Delimited(span, delim, ref tts) => {
                 span.hash_stable(hcx, hasher);
-                std_hash::Hash::hash(&delimited.delim, hasher);
-                for sub_tt in delimited.stream().trees() {
+                std_hash::Hash::hash(&delim, hasher);
+                for sub_tt in tts.stream().trees() {
                     sub_tt.hash_stable(hcx, hasher);
                 }
             }

src/librustc_lint/builtin.rs

@@ -1540,8 +1540,8 @@ impl KeywordIdents {
                     }
                     _ => {},
                 }
-                TokenTree::Delimited(_, ref delim) => {
-                    self.check_tokens(cx, delim.tts.clone().into())
+                TokenTree::Delimited(_, _, tts) => {
+                    self.check_tokens(cx, tts.stream())
                 },
             }
         }

src/libsyntax/ast.rs

@@ -1235,7 +1235,7 @@ pub enum MacDelimiter {
 
 impl Mac_ {
     pub fn stream(&self) -> TokenStream {
-        self.tts.clone().into()
+        self.tts.stream()
     }
 }
 

src/libsyntax/attr/mod.rs

@@ -34,7 +34,7 @@ use parse::token::{self, Token};
 use ptr::P;
 use symbol::Symbol;
 use ThinVec;
-use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
+use tokenstream::{TokenStream, TokenTree, DelimSpan};
 use GLOBALS;
 
 use std::iter;
@@ -549,10 +549,11 @@ impl MetaItemKind {
                     }
                     tokens.push(item.node.tokens());
                 }
-                TokenTree::Delimited(DelimSpan::from_single(span), Delimited {
-                    delim: token::Paren,
-                    tts: TokenStream::concat(tokens).into(),
-                }).into()
+                TokenTree::Delimited(
+                    DelimSpan::from_single(span),
+                    token::Paren,
+                    TokenStream::concat(tokens).into(),
+                ).into()
             }
         }
     }
@@ -570,9 +571,9 @@ impl MetaItemKind {
                     None
                 };
             }
-            Some(TokenTree::Delimited(_, ref delimited)) if delimited.delim == token::Paren => {
+            Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => {
                 tokens.next();
-                delimited.stream()
+                tts.stream()
             }
             _ => return Some(MetaItemKind::Word),
         };

src/libsyntax/ext/expand.rs

@@ -622,9 +622,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
         let mut trees = tokens.trees();
         match trees.next() {
-            Some(TokenTree::Delimited(_, delim)) => {
+            Some(TokenTree::Delimited(_, _, tts)) => {
                 if trees.next().is_none() {
-                    return delim.tts.into()
+                    return tts.into()
                 }
             }
             Some(TokenTree::Token(..)) => {}

src/libsyntax/ext/quote.rs

@@ -36,7 +36,7 @@ pub mod rt {
     use symbol::Symbol;
     use ThinVec;
 
-    use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
+    use tokenstream::{DelimSpan, TokenTree, TokenStream};
 
     pub use parse::new_parser_from_tts;
     pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
@@ -246,9 +246,9 @@ pub mod rt {
             inner.push(self.tokens.clone());
 
             let delim_span = DelimSpan::from_single(self.span);
-            r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
-                delim: token::Bracket, tts: TokenStream::concat(inner).into()
-            }));
+            r.push(TokenTree::Delimited(
+                delim_span, token::Bracket, TokenStream::concat(inner).into()
+            ));
             r
         }
     }
@@ -262,10 +262,9 @@ pub mod rt {
 
     impl ToTokens for () {
         fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
-            vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
-                delim: token::Paren,
-                tts: TokenStream::empty().into(),
-            })]
+            vec![
+                TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into())
+            ]
         }
     }
 
@@ -382,8 +381,6 @@ pub mod rt {
 
 // Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`.
 pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
-    use tokenstream::Delimited;
-
     let mut results = Vec::new();
     let mut result = Vec::new();
     let mut open_span = DUMMY_SP;
@@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
             }
             TokenTree::Token(span, token::CloseDelim(delim)) => {
                 let delim_span = DelimSpan::from_pair(open_span, span);
-                let tree = TokenTree::Delimited(delim_span, Delimited {
+                let tree = TokenTree::Delimited(
+                    delim_span,
                     delim,
-                    tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
-                });
+                    result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
+                );
                 result = results.pop().unwrap();
                 result.push(tree);
             }
@@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
                                     vec![e_tok]);
             vec![cx.stmt_expr(e_push)]
         },
-        TokenTree::Delimited(span, ref delimed) => {
-            let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
-            stmts.extend(statements_mk_tts(cx, delimed.stream()));
-            stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
+        TokenTree::Delimited(span, delim, ref tts) => {
+            let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false);
+            stmts.extend(statements_mk_tts(cx, tts.stream()));
+            stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false));
             stmts
         }
     }

src/libsyntax/ext/tt/quoted.rs

@@ -280,17 +280,17 @@ where
         // `tree` is a `$` token. Look at the next token in `trees`
         tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
             // `tree` is followed by a delimited set of token trees. This indicates the beginning
-            // of a repetition sequence in the macro (e.g., `$(pat)*`).
-            Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
+            // of a repetition sequence in the macro (e.g. `$(pat)*`).
+            Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
                 // Must have `(` not `{` or `[`
-                if delimited.delim != token::Paren {
-                    let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
+                if delim != token::Paren {
+                    let tok = pprust::token_to_string(&token::OpenDelim(delim));
                     let msg = format!("expected `(`, found `{}`", tok);
                     sess.span_diagnostic.span_err(span.entire(), &msg);
                 }
                 // Parse the contents of the sequence itself
                 let sequence = parse(
-                    delimited.tts.into(),
+                    tts.into(),
                     expect_matchers,
                     sess,
                     features,
@@ -354,12 +354,12 @@ where
 
         // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
         // descend into the delimited set and further parse it.
-        tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
+        tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
             span,
             Lrc::new(Delimited {
-                delim: delimited.delim,
+                delim: delim,
                 tts: parse(
-                    delimited.tts.into(),
+                    tts.into(),
                     expect_matchers,
                     sess,
                     features,

src/libsyntax/ext/tt/transcribe.rs

@@ -17,7 +17,7 @@ use fold::noop_fold_tt;
 use parse::token::{self, Token, NtTT};
 use smallvec::SmallVec;
 use syntax_pos::DUMMY_SP;
-use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
+use tokenstream::{TokenStream, TokenTree, DelimSpan};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
@@ -105,10 +105,11 @@ pub fn transcribe(cx: &ExtCtxt,
                     if result_stack.is_empty() {
                         return TokenStream::concat(result);
                     }
-                    let tree = TokenTree::Delimited(span, Delimited {
-                        delim: forest.delim,
-                        tts: TokenStream::concat(result).into(),
-                    });
+                    let tree = TokenTree::Delimited(
+                        span,
+                        forest.delim,
+                        TokenStream::concat(result).into(),
+                    );
                     result = result_stack.pop().unwrap();
                     result.push(tree.into());
                 }

src/libsyntax/fold.rs

@@ -605,12 +605,10 @@ pub fn noop_fold_tt<T: Folder>(tt: TokenTree, fld: &mut T) -> TokenTree {
     match tt {
         TokenTree::Token(span, tok) =>
             TokenTree::Token(fld.new_span(span), fld.fold_token(tok)),
-        TokenTree::Delimited(span, delimed) => TokenTree::Delimited(
+        TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
             DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)),
-            Delimited {
-                tts: fld.fold_tts(delimed.stream()).into(),
-                delim: delimed.delim,
-            }
+            delim,
+            fld.fold_tts(tts.stream()).into(),
         ),
     }
 }

src/libsyntax/parse/lexer/tokentrees.rs

@@ -11,7 +11,7 @@
 use print::pprust::token_to_string;
 use parse::lexer::StringReader;
 use parse::{token, PResult};
-use tokenstream::{Delimited, DelimSpan, TokenStream, TokenTree};
+use tokenstream::{DelimSpan, TokenStream, TokenTree};
 
 impl<'a> StringReader<'a> {
     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
@@ -155,10 +155,11 @@ impl<'a> StringReader<'a> {
                     _ => {}
                 }
 
-                Ok(TokenTree::Delimited(delim_span, Delimited {
+                Ok(TokenTree::Delimited(
+                    delim_span,
                     delim,
-                    tts: tts.into(),
-                }).into())
+                    tts.into(),
+                ).into())
             },
             token::CloseDelim(_) => {
                 // An unexpected closing delimiter (i.e., there is no

src/libsyntax/parse/mod.rs

@@ -764,7 +764,7 @@ mod tests {
     use attr::first_attr_value_str_by_name;
     use parse;
     use print::pprust::item_to_string;
-    use tokenstream::{self, DelimSpan, TokenTree};
+    use tokenstream::{DelimSpan, TokenTree};
     use util::parser_testing::string_to_stream;
     use util::parser_testing::{string_to_expr, string_to_item};
     use with_globals;
@@ -795,42 +795,41 @@ mod tests {
                     Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))),
                     Some(&TokenTree::Token(_, token::Not)),
                     Some(&TokenTree::Token(_, token::Ident(name_zip, false))),
-                    Some(&TokenTree::Delimited(_, ref macro_delimed)),
+                    Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)),
                 )
                 if name_macro_rules.name == "macro_rules"
                 && name_zip.name == "zip" => {
-                    let tts = &macro_delimed.stream().trees().collect::<Vec<_>>();
+                    let tts = &macro_tts.stream().trees().collect::<Vec<_>>();
                     match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) {
                         (
                             3,
-                            Some(&TokenTree::Delimited(_, ref first_delimed)),
+                            Some(&TokenTree::Delimited(_, first_delim, ref first_tts)),
                             Some(&TokenTree::Token(_, token::FatArrow)),
-                            Some(&TokenTree::Delimited(_, ref second_delimed)),
+                            Some(&TokenTree::Delimited(_, second_delim, ref second_tts)),
                         )
-                        if macro_delimed.delim == token::Paren => {
-                            let tts = &first_delimed.stream().trees().collect::<Vec<_>>();
+                        if macro_delim == token::Paren => {
+                            let tts = &first_tts.stream().trees().collect::<Vec<_>>();
                             match (tts.len(), tts.get(0), tts.get(1)) {
                                 (
                                     2,
                                     Some(&TokenTree::Token(_, token::Dollar)),
                                     Some(&TokenTree::Token(_, token::Ident(ident, false))),
                                 )
-                                if first_delimed.delim == token::Paren && ident.name == "a" => {},
-                                _ => panic!("value 3: {:?}", *first_delimed),
+                                if first_delim == token::Paren && ident.name == "a" => {},
+                                _ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
                             }
-                            let tts = &second_delimed.stream().trees().collect::<Vec<_>>();
+                            let tts = &second_tts.stream().trees().collect::<Vec<_>>();
                             match (tts.len(), tts.get(0), tts.get(1)) {
                                 (
                                     2,
                                     Some(&TokenTree::Token(_, token::Dollar)),
                                     Some(&TokenTree::Token(_, token::Ident(ident, false))),
                                 )
-                                if second_delimed.delim == token::Paren
-                                && ident.name == "a" => {},
-                                _ => panic!("value 4: {:?}", *second_delimed),
+                                if second_delim == token::Paren && ident.name == "a" => {},
+                                _ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
                             }
                         },
-                        _ => panic!("value 2: {:?}", *macro_delimed),
+                        _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
                     }
                 },
                 _ => panic!("value: {:?}",tts),
@@ -848,26 +847,24 @@ mod tests {
                 TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
-                    tokenstream::Delimited {
-                        delim: token::DelimToken::Paren,
-                        tts: TokenStream::concat(vec![
-                            TokenTree::Token(sp(6, 7),
-                                             token::Ident(Ident::from_str("b"), false)).into(),
-                            TokenTree::Token(sp(8, 9), token::Colon).into(),
-                            TokenTree::Token(sp(10, 13),
-                                             token::Ident(Ident::from_str("i32"), false)).into(),
-                        ]).into(),
-                    }).into(),
+                    token::DelimToken::Paren,
+                    TokenStream::concat(vec![
+                        TokenTree::Token(sp(6, 7),
+                                         token::Ident(Ident::from_str("b"), false)).into(),
+                        TokenTree::Token(sp(8, 9), token::Colon).into(),
+                        TokenTree::Token(sp(10, 13),
+                                         token::Ident(Ident::from_str("i32"), false)).into(),
+                    ]).into(),
+                ).into(),
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
-                    tokenstream::Delimited {
-                        delim: token::DelimToken::Brace,
-                        tts: TokenStream::concat(vec![
-                            TokenTree::Token(sp(17, 18),
-                                             token::Ident(Ident::from_str("b"), false)).into(),
-                            TokenTree::Token(sp(18, 19), token::Semi).into(),
-                        ]).into(),
-                    }).into()
+                    token::DelimToken::Brace,
+                    TokenStream::concat(vec![
+                        TokenTree::Token(sp(17, 18),
+                                         token::Ident(Ident::from_str("b"), false)).into(),
+                        TokenTree::Token(sp(18, 19), token::Semi).into(),
+                    ]).into(),
+                ).into()
             ]);
 
             assert_eq!(tts, expected);