Skip to content

Commit

Permalink
Rollup merge of rust-lang#127233 - nnethercote:parser-cleanups, r=pet…
Browse files Browse the repository at this point in the history
…rochenkov

Some parser cleanups

Cleanups I made while looking closely at this code.

r? `@petrochenkov`
  • Loading branch information
matthiaskrgr authored Jul 2, 2024
2 parents 9a4e4e0 + edeebe6 commit 0108a84
Show file tree
Hide file tree
Showing 8 changed files with 98 additions and 120 deletions.
14 changes: 8 additions & 6 deletions compiler/rustc_ast/src/attr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,12 +204,14 @@ impl Attribute {

pub fn tokens(&self) -> TokenStream {
match &self.kind {
AttrKind::Normal(normal) => normal
.tokens
.as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
.to_attr_token_stream()
.to_tokenstream(),
AttrKind::Normal(normal) => TokenStream::new(
normal
.tokens
.as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
.to_attr_token_stream()
.to_token_trees(),
),
&AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
token::DocComment(comment_kind, self.style, data),
self.span,
Expand Down
49 changes: 18 additions & 31 deletions compiler/rustc_ast/src/tokenstream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ use rustc_data_structures::sync::{self, Lrc};
use rustc_macros::{Decodable, Encodable, HashStable_Generic};
use rustc_serialize::{Decodable, Encodable};
use rustc_span::{sym, Span, SpanDecoder, SpanEncoder, Symbol, DUMMY_SP};
use smallvec::{smallvec, SmallVec};

use std::borrow::Cow;
use std::{cmp, fmt, iter};
Expand Down Expand Up @@ -180,42 +179,33 @@ impl AttrTokenStream {
AttrTokenStream(Lrc::new(tokens))
}

/// Converts this `AttrTokenStream` to a plain `TokenStream`.
/// Converts this `AttrTokenStream` to a plain `Vec<TokenTree>`.
/// During conversion, `AttrTokenTree::Attributes` get 'flattened'
/// back to a `TokenStream` of the form `outer_attr attr_target`.
/// If there are inner attributes, they are inserted into the proper
/// place in the attribute target tokens.
pub fn to_tokenstream(&self) -> TokenStream {
let trees: Vec<_> = self
.0
.iter()
.flat_map(|tree| match &tree {
pub fn to_token_trees(&self) -> Vec<TokenTree> {
let mut res = Vec::with_capacity(self.0.len());
for tree in self.0.iter() {
match tree {
AttrTokenTree::Token(inner, spacing) => {
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
res.push(TokenTree::Token(inner.clone(), *spacing));
}
AttrTokenTree::Delimited(span, spacing, delim, stream) => {
smallvec![TokenTree::Delimited(
res.push(TokenTree::Delimited(
*span,
*spacing,
*delim,
stream.to_tokenstream()
),]
.into_iter()
TokenStream::new(stream.to_token_trees()),
))
}
AttrTokenTree::Attributes(data) => {
let idx = data
.attrs
.partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
let (outer_attrs, inner_attrs) = data.attrs.split_at(idx);

let mut target_tokens: Vec<_> = data
.tokens
.to_attr_token_stream()
.to_tokenstream()
.0
.iter()
.cloned()
.collect();
let mut target_tokens = data.tokens.to_attr_token_stream().to_token_trees();
if !inner_attrs.is_empty() {
let mut found = false;
// Check the last two trees (to account for a trailing semi)
Expand Down Expand Up @@ -251,17 +241,14 @@ impl AttrTokenStream {
"Failed to find trailing delimited group in: {target_tokens:?}"
);
}
let mut flat: SmallVec<[_; 1]> =
SmallVec::with_capacity(target_tokens.len() + outer_attrs.len());
for attr in outer_attrs {
flat.extend(attr.tokens().0.iter().cloned());
res.extend(attr.tokens().0.iter().cloned());
}
flat.extend(target_tokens);
flat.into_iter()
res.extend(target_tokens);
}
})
.collect();
TokenStream::new(trees)
}
}
res
}
}

Expand Down Expand Up @@ -409,8 +396,8 @@ impl PartialEq<TokenStream> for TokenStream {
}

impl TokenStream {
pub fn new(streams: Vec<TokenTree>) -> TokenStream {
TokenStream(Lrc::new(streams))
pub fn new(tts: Vec<TokenTree>) -> TokenStream {
TokenStream(Lrc::new(tts))
}

pub fn is_empty(&self) -> bool {
Expand Down Expand Up @@ -461,7 +448,7 @@ impl TokenStream {
AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)])
};
attr_stream.to_tokenstream()
TokenStream::new(attr_stream.to_token_trees())
}

pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
Expand Down
18 changes: 8 additions & 10 deletions compiler/rustc_builtin_macros/src/cfg_eval.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,14 @@ pub(crate) fn cfg_eval(
lint_node_id: NodeId,
) -> Annotatable {
let features = Some(features);
CfgEval { cfg: &mut StripUnconfigured { sess, features, config_tokens: true, lint_node_id } }
CfgEval(StripUnconfigured { sess, features, config_tokens: true, lint_node_id })
.configure_annotatable(annotatable)
// Since the item itself has already been configured by the `InvocationCollector`,
// we know that fold result vector will contain exactly one element.
.unwrap()
}

struct CfgEval<'a, 'b> {
cfg: &'a mut StripUnconfigured<'b>,
}
struct CfgEval<'a>(StripUnconfigured<'a>);

fn flat_map_annotatable(
vis: &mut impl MutVisitor,
Expand Down Expand Up @@ -125,9 +123,9 @@ fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
res.is_break()
}

impl CfgEval<'_, '_> {
impl CfgEval<'_> {
fn configure<T: HasAttrs + HasTokens>(&mut self, node: T) -> Option<T> {
self.cfg.configure(node)
self.0.configure(node)
}

fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option<Annotatable> {
Expand Down Expand Up @@ -196,7 +194,7 @@ impl CfgEval<'_, '_> {
// Re-parse the tokens, setting the `capture_cfg` flag to save extra information
// to the captured `AttrTokenStream` (specifically, we capture
// `AttrTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`)
let mut parser = Parser::new(&self.cfg.sess.psess, orig_tokens, None);
let mut parser = Parser::new(&self.0.sess.psess, orig_tokens, None);
parser.capture_cfg = true;
match parse_annotatable_with(&mut parser) {
Ok(a) => annotatable = a,
Expand All @@ -212,16 +210,16 @@ impl CfgEval<'_, '_> {
}
}

impl MutVisitor for CfgEval<'_, '_> {
impl MutVisitor for CfgEval<'_> {
#[instrument(level = "trace", skip(self))]
fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
self.cfg.configure_expr(expr, false);
self.0.configure_expr(expr, false);
mut_visit::noop_visit_expr(expr, self);
}

#[instrument(level = "trace", skip(self))]
fn visit_method_receiver_expr(&mut self, expr: &mut P<ast::Expr>) {
self.cfg.configure_expr(expr, true);
self.0.configure_expr(expr, true);
mut_visit::noop_visit_expr(expr, self);
}

Expand Down
12 changes: 6 additions & 6 deletions compiler/rustc_expand/src/mbe/diagnostics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,21 +120,21 @@ struct CollectTrackerAndEmitter<'a, 'cx, 'matcher> {

struct BestFailure {
token: Token,
position_in_tokenstream: usize,
position_in_tokenstream: u32,
msg: &'static str,
remaining_matcher: MatcherLoc,
}

impl BestFailure {
fn is_better_position(&self, position: usize) -> bool {
fn is_better_position(&self, position: u32) -> bool {
position > self.position_in_tokenstream
}
}

impl<'a, 'cx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'a, 'cx, 'matcher> {
type Failure = (Token, usize, &'static str);
type Failure = (Token, u32, &'static str);

fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure {
fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure {
(tok, position, msg)
}

Expand Down Expand Up @@ -211,9 +211,9 @@ impl<'matcher> FailureForwarder<'matcher> {
}

impl<'matcher> Tracker<'matcher> for FailureForwarder<'matcher> {
type Failure = (Token, usize, &'static str);
type Failure = (Token, u32, &'static str);

fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure {
fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure {
(tok, position, msg)
}

Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_expand/src/mbe/macro_parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,7 @@ impl TtParser {
&mut self,
matcher: &'matcher [MatcherLoc],
token: &Token,
approx_position: usize,
approx_position: u32,
track: &mut T,
) -> Option<NamedParseResult<T::Failure>> {
// Matcher positions that would be valid if the macro invocation was over now. Only
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_expand/src/mbe/macro_rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ pub(super) trait Tracker<'matcher> {
/// Arm failed to match. If the token is `token::Eof`, it indicates an unexpected
/// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
/// The usize is the approximate position of the token in the input token stream.
fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure;
fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure;

/// This is called before trying to match next MatcherLoc on the current token.
fn before_match_loc(&mut self, _parser: &TtParser, _matcher: &'matcher MatcherLoc) {}
Expand Down Expand Up @@ -182,7 +182,7 @@ pub(super) struct NoopTracker;
impl<'matcher> Tracker<'matcher> for NoopTracker {
type Failure = ();

fn build_failure(_tok: Token, _position: usize, _msg: &'static str) -> Self::Failure {}
fn build_failure(_tok: Token, _position: u32, _msg: &'static str) -> Self::Failure {}

fn description() -> &'static str {
"none"
Expand Down
Loading

0 comments on commit 0108a84

Please sign in to comment.