Auto merge of #134161 - nnethercote:overhaul-token-cursors, r=<try>

Overhaul token cursors

r? `@ghost`

compiler/rustc_ast/src/attr/mod.rs

@@ -1,6 +1,5 @@
 //! Functions dealing with attributes and meta items.
 
-use std::iter;
 use std::sync::atomic::{AtomicU32, Ordering};
 
 use rustc_index::bit_set::GrowableBitSet;
@@ -16,7 +15,9 @@ use crate::ast::{
 };
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter, Token};
-use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenTree};
+use crate::tokenstream::{
+    DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree,
+};
 use crate::util::comments;
 use crate::util::literal::escape_string_symbol;
 
@@ -368,22 +369,19 @@ impl MetaItem {
         }
     }
 
-    fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
-    where
-        I: Iterator<Item = &'a TokenTree>,
-    {
+    fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItem> {
         // FIXME: Share code with `parse_path`.
-        let tt = tokens.next().map(|tt| TokenTree::uninterpolate(tt));
+        let tt = iter.next().map(|tt| TokenTree::uninterpolate(tt));
         let path = match tt.as_deref() {
             Some(&TokenTree::Token(
                 Token { kind: ref kind @ (token::Ident(..) | token::PathSep), span },
                 _,
             )) => 'arm: {
                 let mut segments = if let &token::Ident(name, _) = kind {
                     if let Some(TokenTree::Token(Token { kind: token::PathSep, .. }, _)) =
-                        tokens.peek()
+                        iter.peek()
                     {
-                        tokens.next();
+                        iter.next();
                         thin_vec![PathSegment::from_ident(Ident::new(name, span))]
                     } else {
                         break 'arm Path::from_ident(Ident::new(name, span));
@@ -393,16 +391,16 @@ impl MetaItem {
                 };
                 loop {
                     if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
-                        tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
+                        iter.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
                     {
                         segments.push(PathSegment::from_ident(Ident::new(name, span)));
                     } else {
                         return None;
                     }
                     if let Some(TokenTree::Token(Token { kind: token::PathSep, .. }, _)) =
-                        tokens.peek()
+                        iter.peek()
                     {
-                        tokens.next();
+                        iter.next();
                     } else {
                         break;
                     }
@@ -423,8 +421,8 @@ impl MetaItem {
             }
             _ => return None,
         };
-        let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());
-        let kind = MetaItemKind::from_tokens(tokens)?;
+        let list_closing_paren_pos = iter.peek().map(|tt| tt.span().hi());
+        let kind = MetaItemKind::from_tokens(iter)?;
         let hi = match &kind {
             MetaItemKind::NameValue(lit) => lit.span.hi(),
             MetaItemKind::List(..) => list_closing_paren_pos.unwrap_or(path.span.hi()),
@@ -440,25 +438,23 @@ impl MetaItem {
 
 impl MetaItemKind {
     fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<MetaItemInner>> {
-        let mut tokens = tokens.trees().peekable();
+        let mut iter = tokens.iter();
         let mut result = ThinVec::new();
-        while tokens.peek().is_some() {
-            let item = MetaItemInner::from_tokens(&mut tokens)?;
+        while iter.peek().is_some() {
+            let item = MetaItemInner::from_tokens(&mut iter)?;
             result.push(item);
-            match tokens.next() {
+            match iter.next() {
                 None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
                 _ => return None,
             }
         }
         Some(result)
     }
 
-    fn name_value_from_tokens<'a>(
-        tokens: &mut impl Iterator<Item = &'a TokenTree>,
-    ) -> Option<MetaItemKind> {
-        match tokens.next() {
+    fn name_value_from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
+        match iter.next() {
             Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
-                MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
+                MetaItemKind::name_value_from_tokens(&mut inner_tokens.iter())
             }
             Some(TokenTree::Token(token, _)) => {
                 MetaItemLit::from_token(token).map(MetaItemKind::NameValue)
@@ -467,19 +463,17 @@ impl MetaItemKind {
         }
     }
 
-    fn from_tokens<'a>(
-        tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
-    ) -> Option<MetaItemKind> {
-        match tokens.peek() {
+    fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
+        match iter.peek() {
             Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
                 let inner_tokens = inner_tokens.clone();
-                tokens.next();
+                iter.next();
                 MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
             }
             Some(TokenTree::Delimited(..)) => None,
             Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
-                tokens.next();
-                MetaItemKind::name_value_from_tokens(tokens)
+                iter.next();
+                MetaItemKind::name_value_from_tokens(iter)
             }
             _ => Some(MetaItemKind::Word),
         }
@@ -598,22 +592,19 @@ impl MetaItemInner {
         self.meta_item().is_some()
     }
 
-    fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItemInner>
-    where
-        I: Iterator<Item = &'a TokenTree>,
-    {
-        match tokens.peek() {
+    fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemInner> {
+        match iter.peek() {
             Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => {
-                tokens.next();
+                iter.next();
                 return Some(MetaItemInner::Lit(lit));
             }
             Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
-                tokens.next();
-                return MetaItemInner::from_tokens(&mut inner_tokens.trees().peekable());
+                iter.next();
+                return MetaItemInner::from_tokens(&mut inner_tokens.iter());
             }
             _ => {}
         }
-        MetaItem::from_tokens(tokens).map(MetaItemInner::MetaItem)
+        MetaItem::from_tokens(iter).map(MetaItemInner::MetaItem)
     }
 }
 

compiler/rustc_ast/src/tokenstream.rs

@@ -99,7 +99,7 @@ where
     CTX: crate::HashStableContext,
 {
     fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        for sub_tt in self.trees() {
+        for sub_tt in self.iter() {
             sub_tt.hash_stable(hcx, hasher);
         }
     }
@@ -406,7 +406,7 @@ impl Eq for TokenStream {}
 
 impl PartialEq<TokenStream> for TokenStream {
     fn eq(&self, other: &TokenStream) -> bool {
-        self.trees().eq(other.trees())
+        self.iter().eq(other.iter())
     }
 }
 
@@ -423,24 +423,24 @@ impl TokenStream {
         self.0.len()
     }
 
-    pub fn trees(&self) -> RefTokenTreeCursor<'_> {
-        RefTokenTreeCursor::new(self)
+    pub fn get(&self, index: usize) -> Option<&TokenTree> {
+        self.0.get(index)
     }
 
-    pub fn into_trees(self) -> TokenTreeCursor {
-        TokenTreeCursor::new(self)
+    pub fn iter(&self) -> TokenStreamIter<'_> {
+        TokenStreamIter::new(self)
     }
 
     /// Compares two `TokenStream`s, checking equality without regarding span information.
     pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
-        let mut t1 = self.trees();
-        let mut t2 = other.trees();
-        for (t1, t2) in iter::zip(&mut t1, &mut t2) {
-            if !t1.eq_unspanned(t2) {
+        let mut iter1 = self.iter();
+        let mut iter2 = other.iter();
+        for (tt1, tt2) in iter::zip(&mut iter1, &mut iter2) {
+            if !tt1.eq_unspanned(tt2) {
                 return false;
             }
         }
-        t1.next().is_none() && t2.next().is_none()
+        iter1.next().is_none() && iter2.next().is_none()
     }
 
     /// Create a token stream containing a single token with alone spacing. The
@@ -509,7 +509,7 @@ impl TokenStream {
     #[must_use]
     pub fn flattened(&self) -> TokenStream {
         fn can_skip(stream: &TokenStream) -> bool {
-            stream.trees().all(|tree| match tree {
+            stream.iter().all(|tree| match tree {
                 TokenTree::Token(token, _) => !matches!(
                     token.kind,
                     token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
@@ -522,7 +522,7 @@ impl TokenStream {
             return self.clone();
         }
 
-        self.trees().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
+        self.iter().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
     }
 
     // If `vec` is not empty, try to glue `tt` onto its last token. The return
@@ -665,25 +665,26 @@ impl TokenStream {
     }
 }
 
-/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`
-/// items.
 #[derive(Clone)]
-pub struct RefTokenTreeCursor<'t> {
+pub struct TokenStreamIter<'t> {
     stream: &'t TokenStream,
     index: usize,
 }
 
-impl<'t> RefTokenTreeCursor<'t> {
+impl<'t> TokenStreamIter<'t> {
     fn new(stream: &'t TokenStream) -> Self {
-        RefTokenTreeCursor { stream, index: 0 }
+        TokenStreamIter { stream, index: 0 }
     }
 
-    pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
-        self.stream.0.get(self.index + n)
+    // Peeking could be done via `Peekable`, but most iterators need peeking,
+    // and this is simple and avoids the need to use `peekable` and `Peekable`
+    // at all the use sites.
+    pub fn peek(&self) -> Option<&'t TokenTree> {
+        self.stream.0.get(self.index)
     }
 }
 
-impl<'t> Iterator for RefTokenTreeCursor<'t> {
+impl<'t> Iterator for TokenStreamIter<'t> {
     type Item = &'t TokenTree;
 
     fn next(&mut self) -> Option<&'t TokenTree> {
@@ -694,39 +695,6 @@ impl<'t> Iterator for RefTokenTreeCursor<'t> {
     }
 }
 
-/// Owning by-value iterator over a [`TokenStream`], that produces `&TokenTree`
-/// items.
-///
-/// Doesn't impl `Iterator` because Rust doesn't permit an owning iterator to
-/// return `&T` from `next`; the need for an explicit lifetime in the `Item`
-/// associated type gets in the way. Instead, use `next_ref` (which doesn't
-/// involve associated types) for getting individual elements, or
-/// `RefTokenTreeCursor` if you really want an `Iterator`, e.g. in a `for`
-/// loop.
-#[derive(Clone, Debug)]
-pub struct TokenTreeCursor {
-    pub stream: TokenStream,
-    index: usize,
-}
-
-impl TokenTreeCursor {
-    fn new(stream: TokenStream) -> Self {
-        TokenTreeCursor { stream, index: 0 }
-    }
-
-    #[inline]
-    pub fn next_ref(&mut self) -> Option<&TokenTree> {
-        self.stream.0.get(self.index).map(|tree| {
-            self.index += 1;
-            tree
-        })
-    }
-
-    pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
-        self.stream.0.get(self.index + n)
-    }
-}
-
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub struct DelimSpan {
     pub open: Span,

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -732,7 +732,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
     // E.g. we have seen cases where a proc macro can handle `a :: b` but not
     // `a::b`. See #117433 for some examples.
     fn print_tts(&mut self, tts: &TokenStream, convert_dollar_crate: bool) {
-        let mut iter = tts.trees().peekable();
+        let mut iter = tts.iter().peekable();
         while let Some(tt) = iter.next() {
             let spacing = self.print_tt(tt, convert_dollar_crate);
             if let Some(next) = iter.peek() {

compiler/rustc_builtin_macros/src/concat_idents.rs

@@ -19,7 +19,7 @@ pub(crate) fn expand_concat_idents<'cx>(
     }
 
     let mut res_str = String::new();
-    for (i, e) in tts.trees().enumerate() {
+    for (i, e) in tts.iter().enumerate() {
         if i & 1 == 1 {
             match e {
                 TokenTree::Token(Token { kind: token::Comma, .. }, _) => {}

compiler/rustc_builtin_macros/src/trace_macros.rs

@@ -10,17 +10,17 @@ pub(crate) fn expand_trace_macros(
     sp: Span,
     tt: TokenStream,
 ) -> MacroExpanderResult<'static> {
-    let mut cursor = tt.trees();
+    let mut iter = tt.iter();
     let mut err = false;
-    let value = match &cursor.next() {
+    let value = match iter.next() {
         Some(TokenTree::Token(token, _)) if token.is_keyword(kw::True) => true,
         Some(TokenTree::Token(token, _)) if token.is_keyword(kw::False) => false,
         _ => {
             err = true;
             false
         }
     };
-    err |= cursor.next().is_some();
+    err |= iter.next().is_some();
     if err {
         cx.dcx().emit_err(errors::TraceMacros { span: sp });
     } else {