From 74599cd362076816755f087876949c85d8ed92fc Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Mon, 15 Jun 2020 11:50:27 -0400 Subject: [PATCH] Always capture tokens for `macro_rules!` arguments --- src/librustc_expand/mbe/macro_parser.rs | 29 +++++++++-- src/librustc_parse/parser/mod.rs | 2 +- .../ui/proc-macro/auxiliary/first-second.rs | 20 ++++++++ src/test/ui/proc-macro/auxiliary/recollect.rs | 12 +++++ .../ui/proc-macro/auxiliary/weird-hygiene.rs | 48 +++++++++++++++++++ .../proc-macro/capture-macro-rules-invoke.rs | 22 +++++++++ src/test/ui/proc-macro/macro-rules-derive.rs | 20 ++++++++ .../ui/proc-macro/macro-rules-derive.stderr | 9 ++++ src/test/ui/proc-macro/weird-hygiene.rs | 48 +++++++++++++++++++ 9 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 src/test/ui/proc-macro/auxiliary/first-second.rs create mode 100644 src/test/ui/proc-macro/auxiliary/recollect.rs create mode 100644 src/test/ui/proc-macro/auxiliary/weird-hygiene.rs create mode 100644 src/test/ui/proc-macro/capture-macro-rules-invoke.rs create mode 100644 src/test/ui/proc-macro/macro-rules-derive.rs create mode 100644 src/test/ui/proc-macro/macro-rules-derive.stderr create mode 100644 src/test/ui/proc-macro/weird-hygiene.rs diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs index db8258a77863b..968f7c8e273a3 100644 --- a/src/librustc_expand/mbe/macro_parser.rs +++ b/src/librustc_expand/mbe/macro_parser.rs @@ -866,10 +866,23 @@ fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Result(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { + // Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`) + // needs to have them force-captured here. + // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro, + // which requires having captured tokens available. Since we cannot determine + // in advance whether or not a proc-macro will be (transitively) invoked, + // we always capture tokens for any `Nonterminal` which needs them. Ok(match name { - sym::item => match p.parse_item()? { - Some(i) => token::NtItem(i), - None => return Err(p.struct_span_err(p.token.span, "expected an item keyword")), + sym::item => match p.collect_tokens(|this| this.parse_item())? { + (Some(mut item), tokens) => { + // If we captured tokens during parsing (due to outer attributes), + // use those. + if item.tokens.is_none() { + item.tokens = Some(tokens); + } + token::NtItem(item) + } + (None, _) => return Err(p.struct_span_err(p.token.span, "expected an item keyword")), }, sym::block => token::NtBlock(p.parse_block()?), sym::stmt => match p.parse_stmt()? { @@ -877,7 +890,15 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, None => return Err(p.struct_span_err(p.token.span, "expected a statement")), }, sym::pat => token::NtPat(p.parse_pat(None)?), - sym::expr => token::NtExpr(p.parse_expr()?), + sym::expr => { + let (mut expr, tokens) = p.collect_tokens(|this| this.parse_expr())?; + // If we captured tokens during parsing (due to outer attributes), + // use those. + if expr.tokens.is_none() { + expr.tokens = Some(tokens); + } + token::NtExpr(expr) + } sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), sym::ty => token::NtTy(p.parse_ty()?), // this could be handled like a token, since it is one diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index 47ae92c48bd88..7811d5fb741b2 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -1151,7 +1151,7 @@ impl<'a> Parser<'a> { /// This restriction shouldn't be an issue in practice, /// since this function is used to record the tokens for /// a parsed AST item, which always has matching delimiters. - fn collect_tokens( + pub fn collect_tokens( &mut self, f: impl FnOnce(&mut Self) -> PResult<'a, R>, ) -> PResult<'a, (R, TokenStream)> { diff --git a/src/test/ui/proc-macro/auxiliary/first-second.rs b/src/test/ui/proc-macro/auxiliary/first-second.rs new file mode 100644 index 0000000000000..6331608fbe543 --- /dev/null +++ b/src/test/ui/proc-macro/auxiliary/first-second.rs @@ -0,0 +1,20 @@ +// force-host +// no-prefer-dynamic + +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::{TokenStream, TokenTree, Group, Delimiter}; + +#[proc_macro_attribute] +pub fn first(_attr: TokenStream, item: TokenStream) -> TokenStream { + let tokens: TokenStream = "#[derive(Second)]".parse().unwrap(); + let wrapped = TokenTree::Group(Group::new(Delimiter::None, item.into_iter().collect())); + tokens.into_iter().chain(std::iter::once(wrapped)).collect() +} + +#[proc_macro_derive(Second)] +pub fn second(item: TokenStream) -> TokenStream { + TokenStream::new() +} diff --git a/src/test/ui/proc-macro/auxiliary/recollect.rs b/src/test/ui/proc-macro/auxiliary/recollect.rs new file mode 100644 index 0000000000000..d4494a5aff2e9 --- /dev/null +++ b/src/test/ui/proc-macro/auxiliary/recollect.rs @@ -0,0 +1,12 @@ +// force-host +// no-prefer-dynamic + +#![crate_type = "proc-macro"] + +extern crate proc_macro; +use proc_macro::TokenStream; + +#[proc_macro] +pub fn recollect(tokens: TokenStream) -> TokenStream { + tokens.into_iter().collect() +} diff --git a/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs b/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs new file mode 100644 index 0000000000000..338e436df500f --- /dev/null +++ b/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs @@ -0,0 +1,48 @@ +// force-host +// no-prefer-dynamic + +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::{TokenStream, TokenTree, Group}; + +fn find_my_ident(tokens: TokenStream) -> Option { + for token in tokens { + if let TokenTree::Ident(ident) = &token { + if ident.to_string() == "hidden_ident" { + return Some(vec![token].into_iter().collect()) + } + } else if let TokenTree::Group(g) = token { + if let Some(stream) = find_my_ident(g.stream()) { + return Some(stream) + } + } + } + return None; +} + + +#[proc_macro_derive(WeirdDerive)] +pub fn weird_derive(item: TokenStream) -> TokenStream { + let my_ident = find_my_ident(item).expect("Missing 'my_ident'!"); + let tokens: TokenStream = "call_it!();".parse().unwrap(); + let final_call = tokens.into_iter().map(|tree| { + if let TokenTree::Group(g) = tree { + return Group::new(g.delimiter(), my_ident.clone()).into() + } else { + return tree + } + }).collect(); + final_call +} + +#[proc_macro] +pub fn recollect(item: TokenStream) -> TokenStream { + item.into_iter().collect() +} + +#[proc_macro_attribute] +pub fn recollect_attr(_attr: TokenStream, mut item: TokenStream) -> TokenStream { + item.into_iter().collect() +} diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs new file mode 100644 index 0000000000000..a404ddace9bbe --- /dev/null +++ b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs @@ -0,0 +1,22 @@ +// aux-build:test-macros.rs +// check-pass + +extern crate test_macros; +use test_macros::recollect; + +macro_rules! use_expr { + ($expr:expr) => { + recollect!($expr) + } +} + +#[allow(dead_code)] +struct Foo; +impl Foo { + #[allow(dead_code)] + fn use_self(self) { + drop(use_expr!(self)); + } +} + +fn main() {} diff --git a/src/test/ui/proc-macro/macro-rules-derive.rs b/src/test/ui/proc-macro/macro-rules-derive.rs new file mode 100644 index 0000000000000..5b4d577a1acc4 --- /dev/null +++ b/src/test/ui/proc-macro/macro-rules-derive.rs @@ -0,0 +1,20 @@ +// aux-build:first-second.rs +// FIXME: The spans here are bad, see PR #73084 + +extern crate first_second; +use first_second::*; + +macro_rules! produce_it { + ($name:ident) => { + #[first] //~ ERROR cannot find type + struct $name { + field: MissingType + } + } +} + +produce_it!(MyName); + +fn main() { + println!("Hello, world!"); +} diff --git a/src/test/ui/proc-macro/macro-rules-derive.stderr b/src/test/ui/proc-macro/macro-rules-derive.stderr new file mode 100644 index 0000000000000..4b72d29fe8ae0 --- /dev/null +++ b/src/test/ui/proc-macro/macro-rules-derive.stderr @@ -0,0 +1,9 @@ +error[E0412]: cannot find type `MissingType` in this scope + --> $DIR/macro-rules-derive.rs:9:9 + | +LL | #[first] + | ^^^^^^^^ not found in this scope + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0412`. diff --git a/src/test/ui/proc-macro/weird-hygiene.rs b/src/test/ui/proc-macro/weird-hygiene.rs new file mode 100644 index 0000000000000..3f48191b5b26e --- /dev/null +++ b/src/test/ui/proc-macro/weird-hygiene.rs @@ -0,0 +1,48 @@ +// aux-build:weird-hygiene.rs +// check-pass +// FIXME: This should actually error, see PR #73084 + +#![feature(stmt_expr_attributes)] +#![feature(proc_macro_hygiene)] + +extern crate weird_hygiene; +use weird_hygiene::*; + +macro_rules! other { + ($tokens:expr) => { + macro_rules! call_it { + ($outer_ident:ident) => { + macro_rules! inner { + () => { + $outer_ident; + } + } + } + } + + #[derive(WeirdDerive)] + enum MyEnum { + Value = (stringify!($tokens + hidden_ident), 1).1 + } + + inner!(); + } +} + +macro_rules! invoke_it { + ($token:expr) => { + #[recollect_attr] { + $token; + hidden_ident + } + } +} + +fn main() { + // `other` and `invoke_it` are both macro_rules! macros, + // so it should be impossible for them to ever see `hidden_ident`, + // even if they invoke a proc macro. + let hidden_ident = "Hello1"; + other!(50); + invoke_it!(25); +}