Skip to content

Commit 07c12fa

Browse files
nnethercotennethercote
committed
Merge TokenStreamKind into TokenStream.
Because the distinction provides little value, and removing it cleans up the code quite a bit.
1 parent 43c6ece commit 07c12fa

File tree

1 file changed

+65
-72
lines changed

1 file changed

+65
-72
lines changed

‎src/libsyntax/tokenstream.rs

+65-72
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ impl TokenTree {
123123
}
124124

125125
pub fn joint(self) -> TokenStream {
126-
TokenStream { kind: TokenStreamKind::JointTree(self) }
126+
TokenStream::JointTree(self)
127127
}
128128

129129
/// Returns the opening delimiter as a token tree.
@@ -154,8 +154,11 @@ impl TokenTree {
154154
/// instead of a representation of the abstract syntax tree.
155155
/// Today's `TokenTree`s can still contain AST via `Token::Interpolated` for back-compat.
156156
#[derive(Clone, Debug)]
157-
pub struct TokenStream {
158-
kind: TokenStreamKind,
157+
pub enum TokenStream {
158+
Empty,
159+
Tree(TokenTree),
160+
JointTree(TokenTree),
161+
Stream(Lrc<Vec<TokenStream>>),
159162
}
160163

161164
// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
@@ -167,24 +170,22 @@ impl TokenStream {
167170
/// separating the two arguments with a comma for diagnostic suggestions.
168171
pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> {
169172
// Used to suggest if a user writes `foo!(a b);`
170-
if let TokenStreamKind::Stream(ref stream) = self.kind {
173+
if let TokenStream::Stream(ref stream) = self {
171174
let mut suggestion = None;
172175
let mut iter = stream.iter().enumerate().peekable();
173176
while let Some((pos, ts)) = iter.next() {
174177
if let Some((_, next)) = iter.peek() {
175-
let sp = match (&ts.kind, &next.kind) {
176-
(TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma)), _) |
177-
(_, TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma))) => {
178+
let sp = match (&ts, &next) {
179+
(TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) |
180+
(_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => {
178181
continue;
179182
}
180-
(TokenStreamKind::Tree(TokenTree::Token(sp, _)), _) => *sp,
181-
(TokenStreamKind::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(),
183+
(TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp,
184+
(TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(),
182185
_ => continue,
183186
};
184187
let sp = sp.shrink_to_hi();
185-
let comma = TokenStream {
186-
kind: TokenStreamKind::Tree(TokenTree::Token(sp, token::Comma)),
187-
};
188+
let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma));
188189
suggestion = Some((pos, comma, sp));
189190
}
190191
}
@@ -201,17 +202,9 @@ impl TokenStream {
201202
}
202203
}
203204

204-
#[derive(Clone, Debug)]
205-
enum TokenStreamKind {
206-
Empty,
207-
Tree(TokenTree),
208-
JointTree(TokenTree),
209-
Stream(Lrc<Vec<TokenStream>>),
210-
}
211-
212205
impl From<TokenTree> for TokenStream {
213206
fn from(tt: TokenTree) -> TokenStream {
214-
TokenStream { kind: TokenStreamKind::Tree(tt) }
207+
TokenStream::Tree(tt)
215208
}
216209
}
217210

@@ -230,22 +223,22 @@ impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {
230223
impl Extend<TokenStream> for TokenStream {
231224
fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, iter: I) {
232225
let iter = iter.into_iter();
233-
let kind = mem::replace(&mut self.kind, TokenStreamKind::Empty);
226+
let this = mem::replace(self, TokenStream::Empty);
234227

235228
// Vector of token streams originally in self.
236-
let tts: Vec<TokenStream> = match kind {
237-
TokenStreamKind::Empty => {
229+
let tts: Vec<TokenStream> = match this {
230+
TokenStream::Empty => {
238231
let mut vec = Vec::new();
239232
vec.reserve(iter.size_hint().0);
240233
vec
241234
}
242-
TokenStreamKind::Tree(_) | TokenStreamKind::JointTree(_) => {
235+
TokenStream::Tree(_) | TokenStream::JointTree(_) => {
243236
let mut vec = Vec::new();
244237
vec.reserve(1 + iter.size_hint().0);
245-
vec.push(TokenStream { kind });
238+
vec.push(this);
246239
vec
247240
}
248-
TokenStreamKind::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) {
241+
TokenStream::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) {
249242
Ok(mut vec) => {
250243
// Extend in place using the existing capacity if possible.
251244
// This is the fast path for libraries like `quote` that
@@ -286,20 +279,20 @@ impl PartialEq<TokenStream> for TokenStream {
286279

287280
impl TokenStream {
288281
pub fn len(&self) -> usize {
289-
if let TokenStreamKind::Stream(ref slice) = self.kind {
282+
if let TokenStream::Stream(ref slice) = self {
290283
slice.len()
291284
} else {
292285
0
293286
}
294287
}
295288

296289
pub fn empty() -> TokenStream {
297-
TokenStream { kind: TokenStreamKind::Empty }
290+
TokenStream::Empty
298291
}
299292

300293
pub fn is_empty(&self) -> bool {
301-
match self.kind {
302-
TokenStreamKind::Empty => true,
294+
match self {
295+
TokenStream::Empty => true,
303296
_ => false,
304297
}
305298
}
@@ -313,7 +306,7 @@ impl TokenStream {
313306
}
314307

315308
fn concat_rc_vec(streams: Lrc<Vec<TokenStream>>) -> TokenStream {
316-
TokenStream { kind: TokenStreamKind::Stream(streams) }
309+
TokenStream::Stream(streams)
317310
}
318311

319312
pub fn trees(&self) -> Cursor {
@@ -377,9 +370,9 @@ impl TokenStream {
377370
/// Precondition: `self` consists of a single token tree.
378371
/// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
379372
pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
380-
match self.kind {
381-
TokenStreamKind::Tree(tree) => (tree, false),
382-
TokenStreamKind::JointTree(tree) => (tree, true),
373+
match self {
374+
TokenStream::Tree(tree) => (tree, false),
375+
TokenStream::JointTree(tree) => (tree, true),
383376
_ => unreachable!(),
384377
}
385378
}
@@ -389,9 +382,9 @@ impl TokenStream {
389382
let mut result = Vec::new();
390383
let mut i = 0;
391384
while let Some(stream) = trees.next_as_stream() {
392-
result.push(match stream.kind {
393-
TokenStreamKind::Tree(tree) => f(i, tree).into(),
394-
TokenStreamKind::JointTree(tree) => f(i, tree).joint(),
385+
result.push(match stream {
386+
TokenStream::Tree(tree) => f(i, tree).into(),
387+
TokenStream::JointTree(tree) => f(i, tree).joint(),
395388
_ => unreachable!()
396389
});
397390
i += 1;
@@ -403,29 +396,29 @@ impl TokenStream {
403396
let mut trees = self.into_trees();
404397
let mut result = Vec::new();
405398
while let Some(stream) = trees.next_as_stream() {
406-
result.push(match stream.kind {
407-
TokenStreamKind::Tree(tree) => f(tree).into(),
408-
TokenStreamKind::JointTree(tree) => f(tree).joint(),
399+
result.push(match stream {
400+
TokenStream::Tree(tree) => f(tree).into(),
401+
TokenStream::JointTree(tree) => f(tree).joint(),
409402
_ => unreachable!()
410403
});
411404
}
412405
TokenStream::concat(result)
413406
}
414407

415408
fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {
416-
match self.kind {
417-
TokenStreamKind::Empty => None,
418-
TokenStreamKind::Tree(ref tree) => Some((tree.clone(), false)),
419-
TokenStreamKind::JointTree(ref tree) => Some((tree.clone(), true)),
420-
TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
409+
match self {
410+
TokenStream::Empty => None,
411+
TokenStream::Tree(ref tree) => Some((tree.clone(), false)),
412+
TokenStream::JointTree(ref tree) => Some((tree.clone(), true)),
413+
TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
421414
}
422415
}
423416

424417
fn last_tree_if_joint(&self) -> Option<TokenTree> {
425-
match self.kind {
426-
TokenStreamKind::Empty | TokenStreamKind::Tree(..) => None,
427-
TokenStreamKind::JointTree(ref tree) => Some(tree.clone()),
428-
TokenStreamKind::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(),
418+
match self {
419+
TokenStream::Empty | TokenStream::Tree(..) => None,
420+
TokenStream::JointTree(ref tree) => Some(tree.clone()),
421+
TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(),
429422
}
430423
}
431424
}
@@ -472,7 +465,7 @@ impl TokenStreamBuilder {
472465
}
473466

474467
fn push_all_but_last_tree(&mut self, stream: &TokenStream) {
475-
if let TokenStreamKind::Stream(ref streams) = stream.kind {
468+
if let TokenStream::Stream(ref streams) = stream {
476469
let len = streams.len();
477470
match len {
478471
1 => {}
@@ -484,7 +477,7 @@ impl TokenStreamBuilder {
484477
}
485478

486479
fn push_all_but_first_tree(&mut self, stream: &TokenStream) {
487-
if let TokenStreamKind::Stream(ref streams) = stream.kind {
480+
if let TokenStream::Stream(ref streams) = stream {
488481
let len = streams.len();
489482
match len {
490483
1 => {}
@@ -524,10 +517,10 @@ impl StreamCursor {
524517
if self.index < self.stream.len() {
525518
self.index += 1;
526519
let next = self.stream[self.index - 1].clone();
527-
match next.kind {
528-
TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => return Some(next),
529-
TokenStreamKind::Stream(stream) => self.insert(stream),
530-
TokenStreamKind::Empty => {}
520+
match next {
521+
TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next),
522+
TokenStream::Stream(stream) => self.insert(stream),
523+
TokenStream::Empty => {}
531524
}
532525
} else if let Some((stream, index)) = self.stack.pop() {
533526
self.stream = stream;
@@ -548,20 +541,20 @@ impl Iterator for Cursor {
548541
type Item = TokenTree;
549542

550543
fn next(&mut self) -> Option<TokenTree> {
551-
self.next_as_stream().map(|stream| match stream.kind {
552-
TokenStreamKind::Tree(tree) | TokenStreamKind::JointTree(tree) => tree,
544+
self.next_as_stream().map(|stream| match stream {
545+
TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree,
553546
_ => unreachable!()
554547
})
555548
}
556549
}
557550

558551
impl Cursor {
559552
fn new(stream: TokenStream) -> Self {
560-
Cursor(match stream.kind {
561-
TokenStreamKind::Empty => CursorKind::Empty,
562-
TokenStreamKind::Tree(tree) => CursorKind::Tree(tree, false),
563-
TokenStreamKind::JointTree(tree) => CursorKind::JointTree(tree, false),
564-
TokenStreamKind::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)),
553+
Cursor(match stream {
554+
TokenStream::Empty => CursorKind::Empty,
555+
TokenStream::Tree(tree) => CursorKind::Tree(tree, false),
556+
TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false),
557+
TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)),
565558
})
566559
}
567560

@@ -610,11 +603,11 @@ impl Cursor {
610603
pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
611604
fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> {
612605
for stream in streams {
613-
n = match stream.kind {
614-
TokenStreamKind::Tree(ref tree) | TokenStreamKind::JointTree(ref tree)
606+
n = match stream {
607+
TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree)
615608
if n == 0 => return Ok(tree.clone()),
616-
TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => n - 1,
617-
TokenStreamKind::Stream(ref stream) => match look_ahead(stream, n) {
609+
TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1,
610+
TokenStream::Stream(ref stream) => match look_ahead(stream, n) {
618611
Ok(tree) => return Ok(tree),
619612
Err(n) => n,
620613
},
@@ -660,11 +653,11 @@ impl ThinTokenStream {
660653

661654
impl From<TokenStream> for ThinTokenStream {
662655
fn from(stream: TokenStream) -> ThinTokenStream {
663-
ThinTokenStream(match stream.kind {
664-
TokenStreamKind::Empty => None,
665-
TokenStreamKind::Tree(tree) => Some(Lrc::new(vec![tree.into()])),
666-
TokenStreamKind::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])),
667-
TokenStreamKind::Stream(stream) => Some(stream),
656+
ThinTokenStream(match stream {
657+
TokenStream::Empty => None,
658+
TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])),
659+
TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])),
660+
TokenStream::Stream(stream) => Some(stream),
668661
})
669662
}
670663
}

0 commit comments

Comments
 (0)