Skip to content

Commit 881f05b

Browse files
borsbors
committed
Auto merge of #65455 - nnethercote:avoid-unnecessary-TokenTree-to-TokenStream-conversions, r=<try>
Avoid unnecessary `TokenTree` to `TokenStream` conversions A `TokenStream` contains any number of `TokenTrees`. Therefore, a single `TokenTree` can be promoted to a `TokenStream`. But doing so costs two allocations: one for the single-element `Vec`, and one for the `Lrc`. (An `IsJoint` value also must be added; the default is `NonJoint`.) The current code converts `TokenTree`s to `TokenStream`s unnecessarily in a few places. This PR removes some of these unnecessary conversions, both simplifying the code and speeding it up. r? @petrochenkov
2 parents 237d54f + 08afe1f commit 881f05b

File tree

6 files changed

+55
-41
lines changed

6 files changed

+55
-41
lines changed

‎src/libsyntax/attr/mod.rs‎

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use crate::ptr::P;
2222
use crate::sess::ParseSess;
2323
use crate::symbol::{sym, Symbol};
2424
use crate::ThinVec;
25-
use crate::tokenstream::{TokenStream, TokenTree, DelimSpan};
25+
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
2626
use crate::GLOBALS;
2727

2828
use log::debug;
@@ -340,7 +340,10 @@ impl Attribute {
340340
DUMMY_SP,
341341
);
342342
f(&Attribute {
343-
item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) },
343+
item: AttrItem {
344+
path: meta.path,
345+
tokens: TokenStream::new(meta.kind.token_trees_and_joints(meta.span)),
346+
},
344347
id: self.id,
345348
style: self.style,
346349
is_sugared_doc: true,
@@ -400,12 +403,22 @@ pub fn mk_attr(style: AttrStyle, path: Path, tokens: TokenStream, span: Span) ->
400403

401404
/// Returns an inner attribute with the given value and span.
402405
pub fn mk_attr_inner(item: MetaItem) -> Attribute {
403-
mk_attr(AttrStyle::Inner, item.path, item.kind.tokens(item.span), item.span)
406+
mk_attr(
407+
AttrStyle::Inner,
408+
item.path,
409+
TokenStream::new(item.kind.token_trees_and_joints(item.span)),
410+
item.span,
411+
)
404412
}
405413

406414
/// Returns an outer attribute with the given value and span.
407415
pub fn mk_attr_outer(item: MetaItem) -> Attribute {
408-
mk_attr(AttrStyle::Outer, item.path, item.kind.tokens(item.span), item.span)
416+
mk_attr(
417+
AttrStyle::Outer,
418+
item.path,
419+
TokenStream::new(item.kind.token_trees_and_joints(item.span)),
420+
item.span,
421+
)
409422
}
410423

411424
pub fn mk_sugared_doc_attr(text: Symbol, span: Span) -> Attribute {
@@ -415,7 +428,7 @@ pub fn mk_sugared_doc_attr(text: Symbol, span: Span) -> Attribute {
415428
Attribute {
416429
item: AttrItem {
417430
path: Path::from_ident(Ident::with_dummy_span(sym::doc).with_span_pos(span)),
418-
tokens: MetaItemKind::NameValue(lit).tokens(span),
431+
tokens: TokenStream::new(MetaItemKind::NameValue(lit).token_trees_and_joints(span)),
419432
},
420433
id: mk_attr_id(),
421434
style,
@@ -476,7 +489,7 @@ pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option
476489
}
477490

478491
impl MetaItem {
479-
fn tokens(&self) -> TokenStream {
492+
fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
480493
let mut idents = vec![];
481494
let mut last_pos = BytePos(0 as u32);
482495
for (i, segment) in self.path.segments.iter().enumerate() {
@@ -490,8 +503,8 @@ impl MetaItem {
490503
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
491504
last_pos = segment.ident.span.hi();
492505
}
493-
self.kind.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
494-
TokenStream::new(idents)
506+
idents.extend(self.kind.token_trees_and_joints(self.span));
507+
idents
495508
}
496509

497510
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
@@ -550,27 +563,30 @@ impl MetaItem {
550563
}
551564

552565
impl MetaItemKind {
553-
pub fn tokens(&self, span: Span) -> TokenStream {
566+
pub fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> {
554567
match *self {
555-
MetaItemKind::Word => TokenStream::default(),
568+
MetaItemKind::Word => vec![],
556569
MetaItemKind::NameValue(ref lit) => {
557-
let mut vec = vec![TokenTree::token(token::Eq, span).into()];
558-
lit.tokens().append_to_tree_and_joint_vec(&mut vec);
559-
TokenStream::new(vec)
570+
vec![
571+
TokenTree::token(token::Eq, span).into(),
572+
lit.token_tree().into(),
573+
]
560574
}
561575
MetaItemKind::List(ref list) => {
562576
let mut tokens = Vec::new();
563577
for (i, item) in list.iter().enumerate() {
564578
if i > 0 {
565579
tokens.push(TokenTree::token(token::Comma, span).into());
566580
}
567-
item.tokens().append_to_tree_and_joint_vec(&mut tokens);
581+
tokens.extend(item.token_trees_and_joints())
568582
}
569-
TokenTree::Delimited(
570-
DelimSpan::from_single(span),
571-
token::Paren,
572-
TokenStream::new(tokens).into(),
573-
).into()
583+
vec![
584+
TokenTree::Delimited(
585+
DelimSpan::from_single(span),
586+
token::Paren,
587+
TokenStream::new(tokens).into(),
588+
).into()
589+
]
574590
}
575591
}
576592
}
@@ -616,10 +632,10 @@ impl NestedMetaItem {
616632
}
617633
}
618634

619-
fn tokens(&self) -> TokenStream {
635+
fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
620636
match *self {
621-
NestedMetaItem::MetaItem(ref item) => item.tokens(),
622-
NestedMetaItem::Literal(ref lit) => lit.tokens(),
637+
NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(),
638+
NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()],
623639
}
624640
}
625641

‎src/libsyntax/ext/expand.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1487,7 +1487,10 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
14871487

14881488
let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
14891489
*at = attr::Attribute {
1490-
item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) },
1490+
item: AttrItem {
1491+
path: meta.path,
1492+
tokens: TokenStream::new(meta.kind.token_trees_and_joints(meta.span)),
1493+
},
14911494
span: at.span,
14921495
id: at.id,
14931496
style: at.style,

‎src/libsyntax/parse/attr.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ use crate::tokenstream::{TokenStream, TokenTree};
77
use crate::source_map::Span;
88

99
use log::debug;
10-
use smallvec::smallvec;
1110

1211
#[derive(Debug)]
1312
enum InnerAttributeParsePolicy<'a> {
@@ -193,15 +192,15 @@ impl<'a> Parser<'a> {
193192
is_interpolated_expr = true;
194193
}
195194
}
196-
let tokens = if is_interpolated_expr {
195+
let token_tree = if is_interpolated_expr {
197196
// We need to accept arbitrary interpolated expressions to continue
198197
// supporting things like `doc = $expr` that work on stable.
199198
// Non-literal interpolated expressions are rejected after expansion.
200-
self.parse_token_tree().into()
199+
self.parse_token_tree()
201200
} else {
202-
self.parse_unsuffixed_lit()?.tokens()
201+
self.parse_unsuffixed_lit()?.token_tree()
203202
};
204-
TokenStream::from_streams(smallvec![eq.into(), tokens])
203+
TokenStream::new(vec![eq.into(), token_tree.into()])
205204
} else {
206205
TokenStream::default()
207206
};

‎src/libsyntax/parse/literal.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use crate::ast::{self, Lit, LitKind};
44
use crate::parse::token::{self, Token};
55
use crate::symbol::{kw, sym, Symbol};
6-
use crate::tokenstream::{TokenStream, TokenTree};
6+
use crate::tokenstream::TokenTree;
77

88
use log::debug;
99
use rustc_data_structures::sync::Lrc;
@@ -216,13 +216,13 @@ impl Lit {
216216
Lit { token: kind.to_lit_token(), kind, span }
217217
}
218218

219-
/// Losslessly convert an AST literal into a token stream.
220-
crate fn tokens(&self) -> TokenStream {
219+
/// Losslessly convert an AST literal into a token tree.
220+
crate fn token_tree(&self) -> TokenTree {
221221
let token = match self.token.kind {
222222
token::Bool => token::Ident(self.token.symbol, false),
223223
_ => token::Literal(self.token),
224224
};
225-
TokenTree::token(token, self.span).into()
225+
TokenTree::token(token, self.span)
226226
}
227227
}
228228

‎src/libsyntax/parse/parser.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -285,10 +285,10 @@ impl TokenCursor {
285285
token::NoDelim,
286286
&if doc_comment_style(&name.as_str()) == AttrStyle::Inner {
287287
[TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body]
288-
.iter().cloned().collect::<TokenStream>().into()
288+
.iter().cloned().collect::<TokenStream>()
289289
} else {
290290
[TokenTree::token(token::Pound, sp), body]
291-
.iter().cloned().collect::<TokenStream>().into()
291+
.iter().cloned().collect::<TokenStream>()
292292
},
293293
)));
294294

‎src/libsyntax/tokenstream.rs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,9 @@ impl From<TokenTree> for TreeAndJoint {
202202
}
203203
}
204204

205-
impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {
206-
fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
207-
TokenStream::from_streams(iter.into_iter().map(Into::into).collect::<SmallVec<_>>())
205+
impl iter::FromIterator<TokenTree> for TokenStream {
206+
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
207+
TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndJoint>>())
208208
}
209209
}
210210

@@ -271,10 +271,6 @@ impl TokenStream {
271271
}
272272
}
273273

274-
pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec<TreeAndJoint>) {
275-
vec.extend(self.0.iter().cloned());
276-
}
277-
278274
pub fn trees(&self) -> Cursor {
279275
self.clone().into_trees()
280276
}

0 commit comments

Comments
 (0)