diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
index 0906c25cab361..72abcb03410ec 100644
--- a/src/libsyntax/parse/lexer/tokentrees.rs
+++ b/src/libsyntax/parse/lexer/tokentrees.rs
@@ -11,7 +11,7 @@
use print::pprust::token_to_string;
use parse::lexer::StringReader;
use parse::{token, PResult};
-use tokenstream::{DelimSpan, TokenStream, TokenTree};
+use tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree};
impl<'a> StringReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
@@ -178,8 +178,7 @@ impl<'a> StringReader<'a> {
let raw = self.span_src_raw;
self.real_token();
let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
-
- Ok(if is_joint { tt.joint() } else { tt.into() })
+ Ok(TokenStream::Tree(tt, if is_joint { Joint } else { NonJoint }))
}
}
}
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index c11ef33f931d8..620035413a553 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -123,7 +123,7 @@ impl TokenTree {
}
pub fn joint(self) -> TokenStream {
- TokenStream::JointTree(self)
+ TokenStream::Tree(self, Joint)
}
/// Returns the opening delimiter as a token tree.
@@ -156,8 +156,7 @@ impl TokenTree {
#[derive(Clone, Debug)]
pub enum TokenStream {
Empty,
- Tree(TokenTree),
- JointTree(TokenTree),
+ Tree(TokenTree, IsJoint),
Stream(Lrc<Vec<TokenStream>>),
}
@@ -165,6 +164,14 @@ pub enum TokenStream {
#[cfg(target_arch = "x86_64")]
static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::<TokenStream>() == 32);
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum IsJoint {
+ Joint,
+ NonJoint
+}
+
+use self::IsJoint::*;
+
impl TokenStream {
/// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
/// separating the two arguments with a comma for diagnostic suggestions.
@@ -176,16 +183,16 @@ impl TokenStream {
while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) {
- (TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) |
- (_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => {
- continue;
- }
- (TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp,
- (TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(),
+ (TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint), _) |
+ (_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma), NonJoint))
+ => continue,
+ (TokenStream::Tree(TokenTree::Token(sp, _), NonJoint), _) => *sp,
+ (TokenStream::Tree(TokenTree::Delimited(sp, ..), NonJoint), _) =>
+ sp.entire(),
_ => continue,
};
let sp = sp.shrink_to_hi();
- let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma));
+ let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma), NonJoint);
suggestion = Some((pos, comma, sp));
}
}
@@ -204,7 +211,7 @@ impl TokenStream {
impl From<TokenTree> for TokenStream {
fn from(tt: TokenTree) -> TokenStream {
- TokenStream::Tree(tt)
+ TokenStream::Tree(tt, NonJoint)
}
}
@@ -232,7 +239,7 @@ impl Extend<TokenStream> for TokenStream {
vec.reserve(iter.size_hint().0);
vec
}
- TokenStream::Tree(_) | TokenStream::JointTree(_) => {
+ TokenStream::Tree(..) => {
let mut vec = Vec::new();
vec.reserve(1 + iter.size_hint().0);
vec.push(this);
@@ -367,8 +374,7 @@ impl TokenStream {
/// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
match self {
- TokenStream::Tree(tree) => (tree, false),
- TokenStream::JointTree(tree) => (tree, true),
+ TokenStream::Tree(tree, is_joint) => (tree, is_joint == Joint),
_ => unreachable!(),
}
}
@@ -379,8 +385,7 @@ impl TokenStream {
let mut i = 0;
while let Some(stream) = trees.next_as_stream() {
result.push(match stream {
- TokenStream::Tree(tree) => f(i, tree).into(),
- TokenStream::JointTree(tree) => f(i, tree).joint(),
+ TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(i, tree), is_joint),
_ => unreachable!()
});
i += 1;
@@ -393,27 +398,25 @@ impl TokenStream {
let mut result = Vec::new();
while let Some(stream) = trees.next_as_stream() {
result.push(match stream {
- TokenStream::Tree(tree) => f(tree).into(),
- TokenStream::JointTree(tree) => f(tree).joint(),
+ TokenStream::Tree(tree, is_joint) => TokenStream::Tree(f(tree), is_joint),
_ => unreachable!()
});
}
TokenStream::new(result)
}
- fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {
+ fn first_tree_and_joint(&self) -> Option<(TokenTree, IsJoint)> {
match self {
TokenStream::Empty => None,
- TokenStream::Tree(ref tree) => Some((tree.clone(), false)),
- TokenStream::JointTree(ref tree) => Some((tree.clone(), true)),
+ TokenStream::Tree(ref tree, is_joint) => Some((tree.clone(), *is_joint)),
TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
}
}
fn last_tree_if_joint(&self) -> Option<TokenTree> {
match self {
- TokenStream::Empty | TokenStream::Tree(..) => None,
- TokenStream::JointTree(ref tree) => Some(tree.clone()),
+ TokenStream::Empty | TokenStream::Tree(_, NonJoint) => None,
+ TokenStream::Tree(ref tree, Joint) => Some(tree.clone()),
TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(),
}
}
@@ -437,11 +440,7 @@ impl TokenStreamBuilder {
self.push_all_but_last_tree(&last_stream);
let glued_span = last_span.to(span);
let glued_tt = TokenTree::Token(glued_span, glued_tok);
- let glued_tokenstream = if is_joint {
- glued_tt.joint()
- } else {
- glued_tt.into()
- };
+ let glued_tokenstream = TokenStream::Tree(glued_tt, is_joint);
self.0.push(glued_tokenstream);
self.push_all_but_first_tree(&stream);
return
@@ -491,8 +490,7 @@ pub struct Cursor(CursorKind);
#[derive(Clone)]
enum CursorKind {
Empty,
- Tree(TokenTree, bool /* consumed? */),
- JointTree(TokenTree, bool /* consumed? */),
+ Tree(TokenTree, IsJoint, bool /* consumed? */),
Stream(StreamCursor),
}
@@ -514,9 +512,9 @@ impl StreamCursor {
self.index += 1;
let next = self.stream[self.index - 1].clone();
match next {
- TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next),
- TokenStream::Stream(stream) => self.insert(stream),
TokenStream::Empty => {}
+ TokenStream::Tree(..) => return Some(next),
+ TokenStream::Stream(stream) => self.insert(stream),
}
} else if let Some((stream, index)) = self.stack.pop() {
self.stream = stream;
@@ -538,7 +536,7 @@ impl Iterator for Cursor {
fn next(&mut self) -> Option<TokenTree> {
self.next_as_stream().map(|stream| match stream {
- TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree,
+ TokenStream::Tree(tree, _) => tree,
_ => unreachable!()
})
}
@@ -548,18 +546,15 @@ impl Cursor {
fn new(stream: TokenStream) -> Self {
Cursor(match stream {
TokenStream::Empty => CursorKind::Empty,
- TokenStream::Tree(tree) => CursorKind::Tree(tree, false),
- TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false),
+ TokenStream::Tree(tree, is_joint) => CursorKind::Tree(tree, is_joint, false),
TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)),
})
}
pub fn next_as_stream(&mut self) -> Option<TokenStream> {
let (stream, consumed) = match self.0 {
- CursorKind::Tree(ref tree, ref mut consumed @ false) =>
- (tree.clone().into(), consumed),
- CursorKind::JointTree(ref tree, ref mut consumed @ false) =>
- (tree.clone().joint(), consumed),
+ CursorKind::Tree(ref tree, ref is_joint, ref mut consumed @ false) =>
+ (TokenStream::Tree(tree.clone(), *is_joint), consumed),
CursorKind::Stream(ref mut cursor) => return cursor.next_as_stream(),
_ => return None,
};
@@ -572,7 +567,7 @@ impl Cursor {
match self.0 {
_ if stream.is_empty() => return,
CursorKind::Empty => *self = stream.trees(),
- CursorKind::Tree(_, consumed) | CursorKind::JointTree(_, consumed) => {
+ CursorKind::Tree(_, _, consumed) => {
*self = TokenStream::new(vec![self.original_stream(), stream]).trees();
if consumed {
self.next();
@@ -587,8 +582,8 @@ impl Cursor {
pub fn original_stream(&self) -> TokenStream {
match self.0 {
CursorKind::Empty => TokenStream::empty(),
- CursorKind::Tree(ref tree, _) => tree.clone().into(),
- CursorKind::JointTree(ref tree, _) => tree.clone().joint(),
+ CursorKind::Tree(ref tree, ref is_joint, _) =>
+ TokenStream::Tree(tree.clone(), *is_joint),
CursorKind::Stream(ref cursor) => TokenStream::Stream(
cursor.stack.get(0).cloned().map(|(stream, _)| stream)
.unwrap_or_else(|| cursor.stream.clone())
@@ -600,9 +595,8 @@ impl Cursor {
fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result<TokenTree, usize> {
for stream in streams {
n = match stream {
- TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree)
- if n == 0 => return Ok(tree.clone()),
- TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1,
+ TokenStream::Tree(ref tree, _) if n == 0 => return Ok(tree.clone()),
+ TokenStream::Tree(..) => n - 1,
TokenStream::Stream(ref stream) => match look_ahead(stream, n) {
Ok(tree) => return Ok(tree),
Err(n) => n,
@@ -615,10 +609,8 @@ impl Cursor {
match self.0 {
CursorKind::Empty |
- CursorKind::Tree(_, true) |
- CursorKind::JointTree(_, true) => Err(n),
- CursorKind::Tree(ref tree, false) |
- CursorKind::JointTree(ref tree, false) => look_ahead(&[tree.clone().into()], n),
+ CursorKind::Tree(_, _, true) => Err(n),
+ CursorKind::Tree(ref tree, _, false) => look_ahead(&[tree.clone().into()], n),
CursorKind::Stream(ref cursor) => {
look_ahead(&cursor.stream[cursor.index ..], n).or_else(|mut n| {
for &(ref stream, index) in cursor.stack.iter().rev() {
@@ -651,8 +643,7 @@ impl From<TokenStream> for ThinTokenStream {
fn from(stream: TokenStream) -> ThinTokenStream {
ThinTokenStream(match stream {
TokenStream::Empty => None,
- TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])),
- TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])),
+ TokenStream::Tree(..) => Some(Lrc::new(vec![stream])),
TokenStream::Stream(stream) => Some(stream),
})
}
diff --git a/src/libsyntax_ext/proc_macro_server.rs b/src/libsyntax_ext/proc_macro_server.rs
index a04d6c92b7817..fa41022b7b675 100644
--- a/src/libsyntax_ext/proc_macro_server.rs
+++ b/src/libsyntax_ext/proc_macro_server.rs
@@ -21,7 +21,7 @@ use syntax::ast;
use syntax::ext::base::ExtCtxt;
use syntax::parse::lexer::comments;
use syntax::parse::{self, token, ParseSess};
-use syntax::tokenstream::{self, DelimSpan, TokenStream};
+use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream};
use syntax_pos::hygiene::{SyntaxContext, Transparency};
use syntax_pos::symbol::{keywords, Symbol};
use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
@@ -297,11 +297,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
};
let tree = tokenstream::TokenTree::Token(span, token);
- if joint {
- tree.joint()
- } else {
- tree.into()
- }
+ TokenStream::Tree(tree, if joint { Joint } else { NonJoint })
}
}