Skip to content

Commit

Permalink
Auto merge of #60932 - Centril:macro-at-most-once-2015, r=mark-i-m
Browse files Browse the repository at this point in the history 
Support ? Kleene macro operator in 2015

Closes #56668.

See that issue for rationale and discussion.

Crater will be needed (done in #60932 (comment), zero regressions) and then, if all goes well, FCP (in #60932 (comment)).
  • Loading branch information
@bors
bors committed Jun 9, 2019
2 parents 57e13e0 + 3ba82f7 commit e6e60ef
Show file tree
Hide file tree
Showing 16 changed files with 281 additions and 321 deletions.
204 changes: 15 additions & 189 deletions src/libsyntax/ext/tt/quoted.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use crate::ast::NodeId;
use crate::early_buffered_lints::BufferedEarlyLintId;
use crate::ext::tt::macro_parser;
use crate::feature_gate::Features;
use crate::parse::token::{self, Token, TokenKind};
Expand Down Expand Up @@ -250,19 +249,16 @@ pub fn parse(
/// - `sess`: the parsing session. Any errors will be emitted to this session.
/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use
/// unstable features or not.
fn parse_tree<I>(
fn parse_tree(
tree: tokenstream::TokenTree,
trees: &mut Peekable<I>,
trees: &mut Peekable<impl Iterator<Item = tokenstream::TokenTree>>,
expect_matchers: bool,
sess: &ParseSess,
features: &Features,
attrs: &[ast::Attribute],
edition: Edition,
macro_node_id: NodeId,
) -> TokenTree
where
I: Iterator<Item = tokenstream::TokenTree>,
{
) -> TokenTree {
// Depending on what `tree` is, we could be parsing different parts of a macro
match tree {
// `tree` is a `$` token. Look at the next token in `trees`
Expand All @@ -287,16 +283,7 @@ where
macro_node_id,
);
// Get the Kleene operator and optional separator
let (separator, op) =
parse_sep_and_kleene_op(
trees,
span.entire(),
sess,
features,
attrs,
edition,
macro_node_id,
);
let (separator, op) = parse_sep_and_kleene_op(trees, span.entire(), sess);
// Count the number of captured "names" (i.e., named metavars)
let name_captures = macro_parser::count_names(&sequence);
TokenTree::Sequence(
Expand Down Expand Up @@ -375,10 +362,10 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp
/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
/// - Err(span) if the next token tree is not a token
fn parse_kleene_op<I>(input: &mut I, span: Span) -> Result<Result<(KleeneOp, Span), Token>, Span>
where
I: Iterator<Item = tokenstream::TokenTree>,
{
fn parse_kleene_op(
input: &mut impl Iterator<Item = tokenstream::TokenTree>,
span: Span,
) -> Result<Result<(KleeneOp, Span), Token>, Span> {
match input.next() {
Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) {
Some(op) => Ok(Ok((op, token.span))),
Expand All @@ -403,178 +390,20 @@ where
/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
/// error with the appropriate span is emitted to `sess` and a dummy value is returned.
///
/// N.B., in the 2015 edition, `*` and `+` are the only Kleene operators, and `?` is a separator.
/// In the 2018 edition however, `?` is a Kleene operator, and not a separator.
fn parse_sep_and_kleene_op<I>(
input: &mut Peekable<I>,
span: Span,
sess: &ParseSess,
features: &Features,
attrs: &[ast::Attribute],
edition: Edition,
macro_node_id: NodeId,
) -> (Option<Token>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
match edition {
Edition::Edition2015 => parse_sep_and_kleene_op_2015(
input,
span,
sess,
features,
attrs,
macro_node_id,
),
Edition::Edition2018 => parse_sep_and_kleene_op_2018(input, span, sess, features, attrs),
}
}

// `?` is a separator (with a migration warning) and never a KleeneOp.
fn parse_sep_and_kleene_op_2015<I>(
input: &mut Peekable<I>,
span: Span,
sess: &ParseSess,
_features: &Features,
_attrs: &[ast::Attribute],
macro_node_id: NodeId,
) -> (Option<Token>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
// We basically look at two token trees here, denoted as #1 and #2 below
let span = match parse_kleene_op(input, span) {
// #1 is a `+` or `*` KleeneOp
//
// `?` is ambiguous: it could be a separator (warning) or a Kleene::ZeroOrOne (error), so
// we need to look ahead one more token to be sure.
Ok(Ok((op, _))) if op != KleeneOp::ZeroOrOne => return (None, op),

// #1 is `?` token, but it could be a Kleene::ZeroOrOne (error in 2015) without a separator
// or it could be a `?` separator followed by any Kleene operator. We need to look ahead 1
// token to find out which.
Ok(Ok((op, op1_span))) => {
assert_eq!(op, KleeneOp::ZeroOrOne);

// Lookahead at #2. If it is a KleenOp, then #1 is a separator.
let is_1_sep = if let Some(tokenstream::TokenTree::Token(tok2)) = input.peek() {
kleene_op(tok2).is_some()
} else {
false
};

if is_1_sep {
// #1 is a separator and #2 should be a KleepeOp.
// (N.B. We need to advance the input iterator.)
match parse_kleene_op(input, span) {
// #2 is `?`, which is not allowed as a Kleene op in 2015 edition,
// but is allowed in the 2018 edition.
Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => {
sess.span_diagnostic
.struct_span_err(op2_span, "expected `*` or `+`")
.note("`?` is not a macro repetition operator in the 2015 edition, \
but is accepted in the 2018 edition")
.emit();

// Return a dummy
return (None, KleeneOp::ZeroOrMore);
}

// #2 is a Kleene op, which is the only valid option
Ok(Ok((op, _))) => {
// Warn that `?` as a separator will be deprecated
sess.buffer_lint(
BufferedEarlyLintId::QuestionMarkMacroSep,
op1_span,
macro_node_id,
"using `?` as a separator is deprecated and will be \
a hard error in an upcoming edition",
);

return (Some(Token::new(token::Question, op1_span)), op);
}

// #2 is a random token (this is an error) :(
Ok(Err(_)) => op1_span,

// #2 is not even a token at all :(
Err(_) => op1_span,
}
} else {
// `?` is not allowed as a Kleene op in 2015,
// but is allowed in the 2018 edition
sess.span_diagnostic
.struct_span_err(op1_span, "expected `*` or `+`")
.note("`?` is not a macro repetition operator in the 2015 edition, \
but is accepted in the 2018 edition")
.emit();

// Return a dummy
return (None, KleeneOp::ZeroOrMore);
}
}

// #1 is a separator followed by #2, a KleeneOp
Ok(Err(token)) => match parse_kleene_op(input, token.span) {
// #2 is a `?`, which is not allowed as a Kleene op in 2015 edition,
// but is allowed in the 2018 edition
Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => {
sess.span_diagnostic
.struct_span_err(op2_span, "expected `*` or `+`")
.note("`?` is not a macro repetition operator in the 2015 edition, \
but is accepted in the 2018 edition")
.emit();

// Return a dummy
return (None, KleeneOp::ZeroOrMore);
}

// #2 is a KleeneOp :D
Ok(Ok((op, _))) => return (Some(token), op),

// #2 is a random token :(
Ok(Err(token)) => token.span,

// #2 is not a token at all :(
Err(span) => span,
},

// #1 is not a token
Err(span) => span,
};

sess.span_diagnostic.span_err(span, "expected `*` or `+`");

// Return a dummy
(None, KleeneOp::ZeroOrMore)
}

// `?` is a Kleene op, not a separator
fn parse_sep_and_kleene_op_2018<I>(
input: &mut Peekable<I>,
fn parse_sep_and_kleene_op(
input: &mut Peekable<impl Iterator<Item = tokenstream::TokenTree>>,
span: Span,
sess: &ParseSess,
_features: &Features,
_attrs: &[ast::Attribute],
) -> (Option<Token>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
) -> (Option<Token>, KleeneOp) {
// We basically look at two token trees here, denoted as #1 and #2 below
let span = match parse_kleene_op(input, span) {
// #1 is a `?` (needs feature gate)
Ok(Ok((op, _op1_span))) if op == KleeneOp::ZeroOrOne => {
return (None, op);
}

// #1 is a `+` or `*` KleeneOp
// #1 is a `?`, `+`, or `*` KleeneOp
Ok(Ok((op, _))) => return (None, op),

// #1 is a separator followed by #2, a KleeneOp
Ok(Err(token)) => match parse_kleene_op(input, token.span) {
// #2 is the `?` Kleene op, which does not take a separator (error)
Ok(Ok((op, _op2_span))) if op == KleeneOp::ZeroOrOne => {
Ok(Ok((KleeneOp::ZeroOrOne, _))) => {
// Error!
sess.span_diagnostic.span_err(
token.span,
Expand All @@ -588,11 +417,8 @@ where
// #2 is a KleeneOp :D
Ok(Ok((op, _))) => return (Some(token), op),

// #2 is a random token :(
Ok(Err(token)) => token.span,

// #2 is not a token at all :(
Err(span) => span,
// #2 is a random token or not a token at all :(
Ok(Err(Token { span, .. })) | Err(span) => span,
},

// #1 is not a token
Expand Down
50 changes: 50 additions & 0 deletions src/test/run-pass/macros/macro-at-most-once-rep-2015.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// run-pass

#![allow(unused_mut)]

// Check that when `?` is followed by what looks like a Kleene operator (?, +, and *)
// then that `?` is not interpreted as a separator. In other words, `$(pat)?+` matches `pat +`
// or `+` but does not match `pat` or `pat ? pat`.

// edition:2015

macro_rules! foo {
// Check for `?`.
($($a:ident)? ? $num:expr) => {
foo!($($a)? ; $num);
};
// Check for `+`.
($($a:ident)? + $num:expr) => {
foo!($($a)? ; $num);
};
// Check for `*`.
($($a:ident)? * $num:expr) => {
foo!($($a)? ; $num);
};
// Check for `;`, not a kleene operator.
($($a:ident)? ; $num:expr) => {
let mut x = 0;

$(
x += $a;
)?

assert_eq!(x, $num);
};
}

pub fn main() {
let a = 1;

// Accept 0 repetitions.
foo!( ; 0);
foo!( + 0);
foo!( * 0);
foo!( ? 0);

// Accept 1 repetition.
foo!(a ; 1);
foo!(a + 1);
foo!(a * 1);
foo!(a ? 1);
}
50 changes: 50 additions & 0 deletions src/test/run-pass/macros/macro-at-most-once-rep-2018.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// run-pass

#![allow(unused_mut)]

// Check that when `?` is followed by what looks like a Kleene operator (?, +, and *)
// then that `?` is not interpreted as a separator. In other words, `$(pat)?+` matches `pat +`
// or `+` but does not match `pat` or `pat ? pat`.

// edition:2018

macro_rules! foo {
// Check for `?`.
($($a:ident)? ? $num:expr) => {
foo!($($a)? ; $num);
};
// Check for `+`.
($($a:ident)? + $num:expr) => {
foo!($($a)? ; $num);
};
// Check for `*`.
($($a:ident)? * $num:expr) => {
foo!($($a)? ; $num);
};
// Check for `;`, not a kleene operator.
($($a:ident)? ; $num:expr) => {
let mut x = 0;

$(
x += $a;
)?

assert_eq!(x, $num);
};
}

pub fn main() {
let a = 1;

// Accept 0 repetitions.
foo!( ; 0);
foo!( + 0);
foo!( * 0);
foo!( ? 0);

// Accept 1 repetition.
foo!(a ; 1);
foo!(a + 1);
foo!(a * 1);
foo!(a ? 1);
}
33 changes: 0 additions & 33 deletions src/test/run-pass/macros/macro-at-most-once-rep.rs
View file

This file was deleted.

Loading

0 comments on commit e6e60ef

Please sign in to comment.