Skip to content

Commit 495a779

Browse files
authored
Rollup merge of rust-lang#94368 - c410-f3r:metaaaaaaaaaaaaaaaaaaaaaaaaaaa, r=petrochenkov
[1/2] Implement macro meta-variable expressions See rust-lang#93545 (comment) The logic behind `length`, `index` and `count` was removed but the parsing code is still present, i.e., everything is simply ignored like `ignored`. r? `@petrochenkov`
2 parents 005d3b7 + 8073a88 commit 495a779

17 files changed

+900
-44
lines changed

compiler/rustc_ast/src/util/literal.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ pub enum LitError {
2323

2424
impl LitKind {
2525
/// Converts literal token into a semantic literal.
26-
fn from_lit_token(lit: token::Lit) -> Result<LitKind, LitError> {
26+
pub fn from_lit_token(lit: token::Lit) -> Result<LitKind, LitError> {
2727
let token::Lit { kind, symbol, suffix } = lit;
2828
if suffix.is_some() && !kind.may_have_suffix() {
2929
return Err(LitError::InvalidSuffix);

compiler/rustc_expand/src/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#![feature(crate_visibility_modifier)]
44
#![feature(decl_macro)]
55
#![feature(if_let_guard)]
6+
#![feature(let_chains)]
67
#![feature(let_else)]
78
#![feature(proc_macro_diagnostic)]
89
#![feature(proc_macro_internals)]

compiler/rustc_expand/src/mbe.rs

+10-6
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,17 @@
66
crate mod macro_check;
77
crate mod macro_parser;
88
crate mod macro_rules;
9+
crate mod metavar_expr;
910
crate mod quoted;
1011
crate mod transcribe;
1112

13+
use metavar_expr::MetaVarExpr;
1214
use rustc_ast::token::{self, NonterminalKind, Token, TokenKind};
1315
use rustc_ast::tokenstream::DelimSpan;
14-
16+
use rustc_data_structures::sync::Lrc;
1517
use rustc_span::symbol::Ident;
1618
use rustc_span::Span;
1719

18-
use rustc_data_structures::sync::Lrc;
19-
2020
/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
2121
/// that the delimiter itself might be `NoDelim`.
2222
#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
@@ -73,8 +73,8 @@ enum KleeneOp {
7373
ZeroOrOne,
7474
}
7575

76-
/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
77-
/// are "first-class" token trees. Useful for parsing macros.
76+
/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, `$(...)`,
77+
/// and `${...}` are "first-class" token trees. Useful for parsing macros.
7878
#[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
7979
enum TokenTree {
8080
Token(Token),
@@ -85,6 +85,8 @@ enum TokenTree {
8585
MetaVar(Span, Ident),
8686
/// e.g., `$var:expr`. This is only used in the left hand side of MBE macros.
8787
MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
88+
/// A meta-variable expression inside `${...}`
89+
MetaVarExpr(DelimSpan, MetaVarExpr),
8890
}
8991

9092
impl TokenTree {
@@ -139,7 +141,9 @@ impl TokenTree {
139141
TokenTree::Token(Token { span, .. })
140142
| TokenTree::MetaVar(span, _)
141143
| TokenTree::MetaVarDecl(span, _, _) => span,
142-
TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(),
144+
TokenTree::Delimited(span, _)
145+
| TokenTree::MetaVarExpr(span, _)
146+
| TokenTree::Sequence(span, _) => span.entire(),
143147
}
144148
}
145149

compiler/rustc_expand/src/mbe/macro_check.rs

+4
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,8 @@ fn check_binders(
278278
binders.insert(name, BinderInfo { span, ops: ops.into() });
279279
}
280280
}
281+
// `MetaVarExpr` can not appear in the LHS of a macro arm
282+
TokenTree::MetaVarExpr(..) => {}
281283
TokenTree::Delimited(_, ref del) => {
282284
for tt in &del.tts {
283285
check_binders(sess, node_id, tt, macros, binders, ops, valid);
@@ -335,6 +337,8 @@ fn check_occurrences(
335337
let name = MacroRulesNormalizedIdent::new(name);
336338
check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name);
337339
}
340+
// FIXME(c410-f3r) Check token (https://github.com/rust-lang/rust/issues/93902)
341+
TokenTree::MetaVarExpr(..) => {}
338342
TokenTree::Delimited(_, ref del) => {
339343
check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid);
340344
}

compiler/rustc_expand/src/mbe/macro_parser.rs

+9-4
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ struct MatcherPos<'root, 'tt> {
200200

201201
// This type is used a lot. Make sure it doesn't unintentionally get bigger.
202202
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
203-
rustc_data_structures::static_assert_size!(MatcherPos<'_, '_>, 192);
203+
rustc_data_structures::static_assert_size!(MatcherPos<'_, '_>, 240);
204204

205205
impl<'root, 'tt> MatcherPos<'root, 'tt> {
206206
/// Generates the top-level matcher position in which the "dot" is before the first token of
@@ -321,10 +321,13 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
321321
ms.iter().fold(0, |count, elt| {
322322
count
323323
+ match *elt {
324-
TokenTree::Sequence(_, ref seq) => seq.num_captures,
325324
TokenTree::Delimited(_, ref delim) => count_names(&delim.tts),
326325
TokenTree::MetaVar(..) => 0,
327326
TokenTree::MetaVarDecl(..) => 1,
327+
// FIXME(c410-f3r) MetaVarExpr should be handled instead of being ignored
328+
// https://github.com/rust-lang/rust/issues/9390
329+
TokenTree::MetaVarExpr(..) => 0,
330+
TokenTree::Sequence(_, ref seq) => seq.num_captures,
328331
TokenTree::Token(..) => 0,
329332
}
330333
})
@@ -436,7 +439,9 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
436439
}
437440
Occupied(..) => return Err((sp, format!("duplicated bind name: {}", bind_name))),
438441
},
439-
TokenTree::MetaVar(..) | TokenTree::Token(..) => (),
442+
// FIXME(c410-f3r) MetaVar and MetaVarExpr should be handled instead of being ignored
443+
// https://github.com/rust-lang/rust/issues/9390
444+
TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) | TokenTree::Token(..) => {}
440445
}
441446

442447
Ok(())
@@ -650,7 +655,7 @@ fn inner_parse_loop<'root, 'tt>(
650655
// rules. NOTE that this is not necessarily an error unless _all_ items in
651656
// `cur_items` end up doing this. There may still be some other matchers that do
652657
// end up working out.
653-
TokenTree::Token(..) | TokenTree::MetaVar(..) => {}
658+
TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => {}
654659
}
655660
}
656661
}

compiler/rustc_expand/src/mbe/macro_rules.rs

+16-4
Original file line numberDiff line numberDiff line change
@@ -580,7 +580,10 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
580580
use mbe::TokenTree;
581581
for tt in tts {
582582
match *tt {
583-
TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (),
583+
TokenTree::Token(..)
584+
| TokenTree::MetaVar(..)
585+
| TokenTree::MetaVarDecl(..)
586+
| TokenTree::MetaVarExpr(..) => (),
584587
TokenTree::Delimited(_, ref del) => {
585588
if !check_lhs_no_empty_seq(sess, &del.tts) {
586589
return false;
@@ -669,7 +672,10 @@ impl FirstSets {
669672
let mut first = TokenSet::empty();
670673
for tt in tts.iter().rev() {
671674
match *tt {
672-
TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
675+
TokenTree::Token(..)
676+
| TokenTree::MetaVar(..)
677+
| TokenTree::MetaVarDecl(..)
678+
| TokenTree::MetaVarExpr(..) => {
673679
first.replace_with(tt.clone());
674680
}
675681
TokenTree::Delimited(span, ref delimited) => {
@@ -731,7 +737,10 @@ impl FirstSets {
731737
for tt in tts.iter() {
732738
assert!(first.maybe_empty);
733739
match *tt {
734-
TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
740+
TokenTree::Token(..)
741+
| TokenTree::MetaVar(..)
742+
| TokenTree::MetaVarDecl(..)
743+
| TokenTree::MetaVarExpr(..) => {
735744
first.add_one(tt.clone());
736745
return first;
737746
}
@@ -907,7 +916,10 @@ fn check_matcher_core(
907916
// First, update `last` so that it corresponds to the set
908917
// of NT tokens that might end the sequence `... token`.
909918
match *token {
910-
TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
919+
TokenTree::Token(..)
920+
| TokenTree::MetaVar(..)
921+
| TokenTree::MetaVarDecl(..)
922+
| TokenTree::MetaVarExpr(..) => {
911923
if token_can_be_followed_by_any(token) {
912924
// don't need to track tokens that work with any,
913925
last.replace_with_irrelevant();
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
use rustc_ast::token;
2+
use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree};
3+
use rustc_ast::{LitIntType, LitKind};
4+
use rustc_ast_pretty::pprust;
5+
use rustc_errors::{Applicability, PResult};
6+
use rustc_session::parse::ParseSess;
7+
use rustc_span::symbol::Ident;
8+
use rustc_span::Span;
9+
10+
/// A meta-variable expression, for expansions based on properties of meta-variables.
11+
#[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
12+
crate enum MetaVarExpr {
13+
/// The number of repetitions of an identifier, optionally limited to a number
14+
/// of outer-most repetition depths. If the depth limit is `None` then the depth is unlimited.
15+
Count(Ident, Option<usize>),
16+
17+
/// Ignore a meta-variable for repetition without expansion.
18+
Ignore(Ident),
19+
20+
/// The index of the repetition at a particular depth, where 0 is the inner-most
21+
/// repetition. The `usize` is the depth.
22+
Index(usize),
23+
24+
/// The length of the repetition at a particular depth, where 0 is the inner-most
25+
/// repetition. The `usize` is the depth.
26+
Length(usize),
27+
}
28+
29+
impl MetaVarExpr {
30+
/// Attempt to parse a meta-variable expression from a token stream.
31+
crate fn parse<'sess>(
32+
input: &TokenStream,
33+
outer_span: Span,
34+
sess: &'sess ParseSess,
35+
) -> PResult<'sess, MetaVarExpr> {
36+
let mut tts = input.trees();
37+
let ident = parse_ident(&mut tts, sess, outer_span)?;
38+
let Some(TokenTree::Delimited(_, token::Paren, args)) = tts.next() else {
39+
let msg = "meta-variable expression parameter must be wrapped in parentheses";
40+
return Err(sess.span_diagnostic.struct_span_err(ident.span, msg));
41+
};
42+
check_trailing_token(&mut tts, sess)?;
43+
let mut iter = args.trees();
44+
let rslt = match &*ident.as_str() {
45+
"count" => parse_count(&mut iter, sess, ident.span)?,
46+
"ignore" => MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?),
47+
"index" => MetaVarExpr::Index(parse_depth(&mut iter, sess, ident.span)?),
48+
"length" => MetaVarExpr::Length(parse_depth(&mut iter, sess, ident.span)?),
49+
_ => {
50+
let err_msg = "unrecognized meta-variable expression";
51+
let mut err = sess.span_diagnostic.struct_span_err(ident.span, err_msg);
52+
err.span_suggestion(
53+
ident.span,
54+
"supported expressions are count, ignore, index and length",
55+
String::new(),
56+
Applicability::MachineApplicable,
57+
);
58+
return Err(err);
59+
}
60+
};
61+
check_trailing_token(&mut iter, sess)?;
62+
Ok(rslt)
63+
}
64+
65+
crate fn ident(&self) -> Option<&Ident> {
66+
match self {
67+
MetaVarExpr::Count(ident, _) | MetaVarExpr::Ignore(ident) => Some(&ident),
68+
MetaVarExpr::Index(..) | MetaVarExpr::Length(..) => None,
69+
}
70+
}
71+
}
72+
73+
// Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
74+
fn check_trailing_token<'sess>(iter: &mut Cursor, sess: &'sess ParseSess) -> PResult<'sess, ()> {
75+
if let Some(tt) = iter.next() {
76+
let mut diag = sess.span_diagnostic.struct_span_err(
77+
tt.span(),
78+
&format!("unexpected token: {}", pprust::tt_to_string(&tt)),
79+
);
80+
diag.span_note(tt.span(), "meta-variable expression must not have trailing tokens");
81+
Err(diag)
82+
} else {
83+
Ok(())
84+
}
85+
}
86+
87+
/// Parse a meta-variable `count` expression: `count(ident[, depth])`
88+
fn parse_count<'sess>(
89+
iter: &mut Cursor,
90+
sess: &'sess ParseSess,
91+
span: Span,
92+
) -> PResult<'sess, MetaVarExpr> {
93+
let ident = parse_ident(iter, sess, span)?;
94+
let depth = if try_eat_comma(iter) { Some(parse_depth(iter, sess, span)?) } else { None };
95+
Ok(MetaVarExpr::Count(ident, depth))
96+
}
97+
98+
/// Parses the depth used by index(depth) and length(depth).
99+
fn parse_depth<'sess>(
100+
iter: &mut Cursor,
101+
sess: &'sess ParseSess,
102+
span: Span,
103+
) -> PResult<'sess, usize> {
104+
let Some(tt) = iter.next() else { return Ok(0) };
105+
let TokenTree::Token(token::Token {
106+
kind: token::TokenKind::Literal(lit), ..
107+
}) = tt else {
108+
return Err(sess.span_diagnostic.struct_span_err(
109+
span,
110+
"meta-variable expression depth must be a literal"
111+
));
112+
};
113+
if let Ok(lit_kind) = LitKind::from_lit_token(lit)
114+
&& let LitKind::Int(n_u128, LitIntType::Unsuffixed) = lit_kind
115+
&& let Ok(n_usize) = usize::try_from(n_u128)
116+
{
117+
Ok(n_usize)
118+
}
119+
else {
120+
let msg = "only unsuffixes integer literals are supported in meta-variable expressions";
121+
Err(sess.span_diagnostic.struct_span_err(span, msg))
122+
}
123+
}
124+
125+
/// Parses an generic ident
126+
fn parse_ident<'sess>(
127+
iter: &mut Cursor,
128+
sess: &'sess ParseSess,
129+
span: Span,
130+
) -> PResult<'sess, Ident> {
131+
let err_fn = |msg| sess.span_diagnostic.struct_span_err(span, msg);
132+
if let Some(tt) = iter.next() && let TokenTree::Token(token) = tt {
133+
if let Some((elem, false)) = token.ident() {
134+
return Ok(elem);
135+
}
136+
let token_str = pprust::token_to_string(&token);
137+
let mut err = err_fn(&format!("expected identifier, found `{}`", &token_str));
138+
err.span_suggestion(
139+
token.span,
140+
&format!("try removing `{}`", &token_str),
141+
String::new(),
142+
Applicability::MaybeIncorrect,
143+
);
144+
return Err(err);
145+
}
146+
Err(err_fn("expected identifier"))
147+
}
148+
149+
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
150+
/// iterator is not modified and the result is `false`.
151+
fn try_eat_comma(iter: &mut Cursor) -> bool {
152+
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. })) = iter.look_ahead(0) {
153+
let _ = iter.next();
154+
return true;
155+
}
156+
false
157+
}

0 commit comments

Comments
 (0)