Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove LazyTokenStream. #58476

Merged
merged 5 commits into from
Feb 23, 2019
Merged

Remove LazyTokenStream. #58476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d26bf74
Change `Token::interpolated_to_tokenstream()`.
nnethercote Feb 14, 2019
f8801f3
Remove `LazyTokenStream`.
nnethercote Feb 14, 2019
f0d8fbd
Avoid a `clone()` in `transcribe()`.
nnethercote Feb 15, 2019
82ad4f1
Make `interpolated_to_tokenstream` a method on `Nonterminal`.
nnethercote Feb 17, 2019
895a794
Remove some unnecessary `into()` calls.
nnethercote Feb 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/librustc/hir/lowering.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1124,19 +1124,19 @@ impl<'a> LoweringContext<'a> {
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
span,
delim,
self.lower_token_stream(tts.into()).into(),
self.lower_token_stream(tts),
).into(),
}
}

fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
match token {
Token::Interpolated(_) => {}
other => return TokenTree::Token(span, other).into(),
Token::Interpolated(nt) => {
let tts = nt.to_tokenstream(&self.sess.parse_sess, span);
self.lower_token_stream(tts)
}
other => TokenTree::Token(span, other).into(),
}

let tts = token.interpolated_to_tokenstream(&self.sess.parse_sess, span);
self.lower_token_stream(tts)
}

fn lower_arm(&mut self, arm: &Arm) -> hir::Arm {
Expand Down
2 changes: 1 addition & 1 deletion src/librustc/hir/map/def_collector.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> {

fn visit_token(&mut self, t: Token) {
if let Token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = nt.0 {
if let token::NtExpr(ref expr) = *nt {
if let ExprKind::Mac(..) = expr.node {
self.visit_macro_invoc(expr.id);
}
Expand Down
2 changes: 1 addition & 1 deletion src/librustc_resolve/build_reduced_graph.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1025,7 +1025,7 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> {

fn visit_token(&mut self, t: Token) {
if let Token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = nt.0 {
if let token::NtExpr(ref expr) = *nt {
if let ast::ExprKind::Mac(..) = expr.node {
self.visit_invoc(expr.id);
}
Expand Down
4 changes: 2 additions & 2 deletions src/libsyntax/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ impl MetaItem {
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
Path { span, segments }
}
Some(TokenTree::Token(_, Token::Interpolated(ref nt))) => match nt.0 {
Some(TokenTree::Token(_, Token::Interpolated(nt))) => match *nt {
token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()),
token::Nonterminal::NtPath(ref path) => path.clone(),
Expand Down Expand Up @@ -682,7 +682,7 @@ impl LitKind {
match token {
Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)),
Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)),
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(nt) => match *nt {
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
ExprKind::Lit(ref lit) => Some(lit.node.clone()),
_ => None,
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ext/base.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ impl<F> TTMacroExpander for F
impl MutVisitor for AvoidInterpolatedIdents {
fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) {
if let tokenstream::TokenTree::Token(_, token::Interpolated(nt)) = tt {
if let token::NtIdent(ident, is_raw) = nt.0 {
if let token::NtIdent(ident, is_raw) = **nt {
*tt = tokenstream::TokenTree::Token(ident.span,
token::Ident(ident, is_raw));
}
Expand Down
5 changes: 3 additions & 2 deletions src/libsyntax/ext/expand.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use syntax_pos::{Span, DUMMY_SP, FileName};
use syntax_pos::hygiene::ExpnFormat;

use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
use std::fs;
use std::io::ErrorKind;
use std::{iter, mem};
Expand Down Expand Up @@ -586,14 +587,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
}
AttrProcMacro(ref mac, ..) => {
self.gate_proc_macro_attr_item(attr.span, &item);
let item_tok = TokenTree::Token(DUMMY_SP, Token::interpolated(match item {
let item_tok = TokenTree::Token(DUMMY_SP, Token::Interpolated(Lrc::new(match item {
Annotatable::Item(item) => token::NtItem(item),
Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
Annotatable::Expr(expr) => token::NtExpr(expr),
})).into();
}))).into();
let input = self.extract_proc_macro_attr_input(attr.tokens, attr.span);
let tok_result = mac.expand(self.cx, attr.span, input, item_tok);
let res = self.parse_ast_fragment(tok_result, invoc.fragment_kind,
Expand Down
29 changes: 15 additions & 14 deletions src/libsyntax/ext/tt/macro_parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ use smallvec::{smallvec, SmallVec};
use syntax_pos::Span;

use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::mem;
use std::ops::{Deref, DerefMut};
Expand Down Expand Up @@ -179,7 +180,7 @@ struct MatcherPos<'root, 'tt: 'root> {
/// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep`
/// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one
/// wants the shared `matches`, one should use `up.matches`.
matches: Box<[Rc<NamedMatchVec>]>,
matches: Box<[Lrc<NamedMatchVec>]>,
/// The position in `matches` corresponding to the first metavar in this matcher's sequence of
/// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
/// to `matches[match_lo]`.
Expand Down Expand Up @@ -218,7 +219,7 @@ struct MatcherPos<'root, 'tt: 'root> {
impl<'root, 'tt> MatcherPos<'root, 'tt> {
/// Adds `m` as a named match for the `idx`-th metavar.
fn push_match(&mut self, idx: usize, m: NamedMatch) {
let matches = Rc::make_mut(&mut self.matches[idx]);
let matches = Lrc::make_mut(&mut self.matches[idx]);
matches.push(m);
}
}
Expand Down Expand Up @@ -295,11 +296,11 @@ pub fn count_names(ms: &[TokenTree]) -> usize {
}

/// `len` `Vec`s (initially shared and empty) that will store matches of metavars.
fn create_matches(len: usize) -> Box<[Rc<NamedMatchVec>]> {
fn create_matches(len: usize) -> Box<[Lrc<NamedMatchVec>]> {
if len == 0 {
vec![]
} else {
let empty_matches = Rc::new(SmallVec::new());
let empty_matches = Lrc::new(SmallVec::new());
vec![empty_matches; len]
}.into_boxed_slice()
}
Expand Down Expand Up @@ -353,8 +354,8 @@ fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherP
/// token tree it was derived from.
#[derive(Debug, Clone)]
pub enum NamedMatch {
MatchedSeq(Rc<NamedMatchVec>, DelimSpan),
MatchedNonterminal(Rc<Nonterminal>),
MatchedSeq(Lrc<NamedMatchVec>, DelimSpan),
MatchedNonterminal(Lrc<Nonterminal>),
}

/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input
Expand Down Expand Up @@ -561,7 +562,7 @@ fn inner_parse_loop<'root, 'tt>(
new_item.match_cur += seq.num_captures;
new_item.idx += 1;
for idx in item.match_cur..item.match_cur + seq.num_captures {
new_item.push_match(idx, MatchedSeq(Rc::new(smallvec![]), sp));
new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp));
}
cur_items.push(new_item);
}
Expand Down Expand Up @@ -707,7 +708,7 @@ pub fn parse(
let matches = eof_items[0]
.matches
.iter_mut()
.map(|dv| Rc::make_mut(dv).pop().unwrap());
.map(|dv| Lrc::make_mut(dv).pop().unwrap());
return nameize(sess, ms, matches);
} else if eof_items.len() > 1 {
return Error(
Expand Down Expand Up @@ -780,7 +781,7 @@ pub fn parse(
let match_cur = item.match_cur;
item.push_match(
match_cur,
MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.as_str()))),
MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, &ident.as_str()))),
);
item.idx += 1;
item.match_cur += 1;
Expand Down Expand Up @@ -829,7 +830,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
},
"block" => match *token {
Token::OpenDelim(token::Brace) => true,
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(ref nt) => match **nt {
token::NtItem(_)
| token::NtPat(_)
| token::NtTy(_)
Expand All @@ -843,9 +844,9 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
},
"path" | "meta" => match *token {
Token::ModSep | Token::Ident(..) => true,
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(ref nt) => match **nt {
token::NtPath(_) | token::NtMeta(_) => true,
_ => may_be_ident(&nt.0),
_ => may_be_ident(&nt),
},
_ => false,
},
Expand All @@ -862,12 +863,12 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
Token::ModSep | // path
Token::Lt | // path (UFCS constant)
Token::BinOp(token::Shl) => true, // path (double UFCS)
Token::Interpolated(ref nt) => may_be_ident(&nt.0),
Token::Interpolated(ref nt) => may_be_ident(nt),
_ => false,
},
"lifetime" => match *token {
Token::Lifetime(_) => true,
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(ref nt) => match **nt {
token::NtLifetime(_) | token::NtTT(_) => true,
_ => false,
},
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ext/tt/transcribe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ pub fn transcribe(cx: &ExtCtxt<'_>,
result.push(tt.clone().into());
} else {
sp = sp.apply_mark(cx.current_expansion.mark);
let token = TokenTree::Token(sp, Token::interpolated((**nt).clone()));
let token = TokenTree::Token(sp, Token::Interpolated(nt.clone()));
result.push(token.into());
}
} else {
Expand Down
5 changes: 2 additions & 3 deletions src/libsyntax/mut_visit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,9 +581,8 @@ pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
token::Ident(id, _is_raw) => vis.visit_ident(id),
token::Lifetime(id) => vis.visit_ident(id),
token::Interpolated(nt) => {
let nt = Lrc::make_mut(nt);
vis.visit_interpolated(&mut nt.0);
nt.1 = token::LazyTokenStream::new();
let mut nt = Lrc::make_mut(nt);
vis.visit_interpolated(&mut nt);
}
_ => {}
}
Expand Down
4 changes: 2 additions & 2 deletions src/libsyntax/parse/attr.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ impl<'a> Parser<'a> {
/// The delimiters or `=` are still put into the resulting token stream.
crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
let meta = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
_ => None,
},
Expand Down Expand Up @@ -227,7 +227,7 @@ impl<'a> Parser<'a> {
/// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
let nt_meta = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtMeta(ref e) => Some(e.clone()),
_ => None,
},
Expand Down
16 changes: 8 additions & 8 deletions src/libsyntax/parse/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ enum BlockMode {
macro_rules! maybe_whole_expr {
($p:expr) => {
if let token::Interpolated(nt) = $p.token.clone() {
match nt.0 {
match *nt {
token::NtExpr(ref e) | token::NtLiteral(ref e) => {
$p.bump();
return Ok((*e).clone());
Expand All @@ -146,7 +146,7 @@ macro_rules! maybe_whole_expr {
macro_rules! maybe_whole {
($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
if let token::Interpolated(nt) = $p.token.clone() {
if let token::$constructor($x) = nt.0.clone() {
if let token::$constructor($x) = (*nt).clone() {
$p.bump();
return Ok($e);
}
Expand Down Expand Up @@ -1570,7 +1570,7 @@ impl<'a> Parser<'a> {
Some(body)
}
token::Interpolated(ref nt) => {
match &nt.0 {
match **nt {
token::NtBlock(..) => {
*at_end = true;
let (inner_attrs, body) = self.parse_inner_attrs_and_block()?;
Expand Down Expand Up @@ -1913,7 +1913,7 @@ impl<'a> Parser<'a> {

fn is_named_argument(&mut self) -> bool {
let offset = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon),
_ => 0,
}
Expand Down Expand Up @@ -2099,7 +2099,7 @@ impl<'a> Parser<'a> {
/// Matches `token_lit = LIT_INTEGER | ...`.
fn parse_lit_token(&mut self) -> PResult<'a, LitKind> {
let out = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
ExprKind::Lit(ref lit) => { lit.node.clone() }
_ => { return self.unexpected_last(&self.token); }
Expand Down Expand Up @@ -2299,7 +2299,7 @@ impl<'a> Parser<'a> {
/// attributes.
pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, ast::Path> {
let meta_ident = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtMeta(ref meta) => match meta.node {
ast::MetaItemKind::Word => Some(meta.ident.clone()),
_ => None,
Expand Down Expand Up @@ -3271,7 +3271,7 @@ impl<'a> Parser<'a> {
self.meta_var_span = Some(self.span);
// Interpolated identifier and lifetime tokens are replaced with usual identifier
// and lifetime tokens, so the former are never encountered during normal parsing.
match nt.0 {
match **nt {
token::NtIdent(ident, is_raw) => (token::Ident(ident, is_raw), ident.span),
token::NtLifetime(ident) => (token::Lifetime(ident), ident.span),
_ => return,
Expand Down Expand Up @@ -3403,7 +3403,7 @@ impl<'a> Parser<'a> {
// can't continue an expression after an ident
token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw),
token::Literal(..) | token::Pound => true,
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtIdent(..) | token::NtExpr(..) |
token::NtBlock(..) | token::NtPath(..) => true,
_ => false,
Expand Down
Loading