From 858ad51d85de4bfac42e3d2121c7ee839b940337 Mon Sep 17 00:00:00 2001 From: Asuna Date: Sat, 4 Jan 2025 02:53:49 +0100 Subject: [PATCH] Append `TokenTree` with `ToTokens` in `proc_macro::quote!` --- library/proc_macro/src/quote.rs | 82 +++++++++++-------- .../tests/ui/auxiliary/proc_macro_attr.rs | 2 +- .../tests/ui/auxiliary/proc_macro_derive.rs | 2 +- .../deriving/auxiliary/another-proc-macro.rs | 1 + tests/ui/hygiene/auxiliary/opaque-hygiene.rs | 1 + tests/ui/macros/auxiliary/hello_macro.rs | 1 + tests/ui/macros/auxiliary/issue-100199.rs | 3 +- tests/ui/macros/auxiliary/proc_macro_def.rs | 1 + .../macros/auxiliary/proc_macro_sequence.rs | 4 +- tests/ui/proc-macro/auxiliary/cond_plugin.rs | 1 + .../auxiliary/count_compound_ops.rs | 3 +- tests/ui/proc-macro/auxiliary/double.rs | 1 + .../auxiliary/generate-dollar-ident.rs | 1 + .../auxiliary/hygiene_example_codegen.rs | 1 + .../proc-macro/auxiliary/mixed-site-span.rs | 7 +- .../auxiliary/nonterminal-recollect-attr.rs | 1 + .../auxiliary/resolved-located-at.rs | 1 + .../auxiliary/span-from-proc-macro.rs | 7 +- tests/ui/proc-macro/quote-debug.stdout | 42 +++++----- tests/ui/proc-macro/quote-interpolation.rs | 16 ++++ .../ui/proc-macro/span-from-proc-macro.stderr | 8 +- .../auxiliary/proc-macro-type-error.rs | 3 +- 22 files changed, 119 insertions(+), 70 deletions(-) create mode 100644 tests/ui/proc-macro/quote-interpolation.rs diff --git a/library/proc_macro/src/quote.rs b/library/proc_macro/src/quote.rs index 6faa41b297005..c0c5d3d732c99 100644 --- a/library/proc_macro/src/quote.rs +++ b/library/proc_macro/src/quote.rs @@ -4,7 +4,9 @@ //! This quasiquoter uses macros 2.0 hygiene to reliably access //! items from `proc_macro`, to build a `proc_macro::TokenStream`. -use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use crate::{ + Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree, +}; macro_rules! minimal_quote_tt { (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, minimal_quote!($($t)*)) }; @@ -50,7 +52,7 @@ macro_rules! minimal_quote { () => { TokenStream::new() }; ($($t:tt)*) => { [ - $(TokenStream::from(minimal_quote_ts!($t)),)* + $(ToTokens::into_token_stream(minimal_quote_ts!($t)),)* ].iter().cloned().collect::() }; } @@ -66,35 +68,39 @@ pub fn quote(stream: TokenStream) -> TokenStream { } let proc_macro_crate = minimal_quote!(crate); let mut after_dollar = false; - let tokens = stream - .into_iter() - .filter_map(|tree| { - if after_dollar { - after_dollar = false; - match tree { - TokenTree::Ident(_) => { - return Some(minimal_quote!(Into::::into( - Clone::clone(&(@ tree))),)); - } - TokenTree::Punct(ref tt) if tt.as_char() == '$' => {} - _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), - } - } else if let TokenTree::Punct(ref tt) = tree { - if tt.as_char() == '$' { - after_dollar = true; - return None; + + let mut tokens = crate::TokenStream::new(); + for tree in stream { + if after_dollar { + after_dollar = false; + match tree { + TokenTree::Ident(_) => { + minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);) + .to_tokens(&mut tokens); + continue; } + TokenTree::Punct(ref tt) if tt.as_char() == '$' => {} + _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), } + } else if let TokenTree::Punct(ref tt) = tree { + if tt.as_char() == '$' { + after_dollar = true; + continue; + } + } - Some(minimal_quote!(crate::TokenStream::from((@ match tree { - TokenTree::Punct(tt) => minimal_quote!(crate::TokenTree::Punct(crate::Punct::new( + match tree { + TokenTree::Punct(tt) => { + minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new( (@ TokenTree::from(Literal::character(tt.as_char()))), (@ match tt.spacing() { Spacing::Alone => minimal_quote!(crate::Spacing::Alone), Spacing::Joint => minimal_quote!(crate::Spacing::Joint), }), - ))), - TokenTree::Group(tt) => minimal_quote!(crate::TokenTree::Group(crate::Group::new( + )), &mut ts);) + } + TokenTree::Group(tt) => { + minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new( (@ match tt.delimiter() { Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis), Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace), @@ -102,12 +108,16 @@ pub fn quote(stream: TokenStream) -> TokenStream { Delimiter::None => minimal_quote!(crate::Delimiter::None), }), (@ quote(tt.stream())), - ))), - TokenTree::Ident(tt) => minimal_quote!(crate::TokenTree::Ident(crate::Ident::new( + )), &mut ts);) + } + TokenTree::Ident(tt) => { + minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new( (@ TokenTree::from(Literal::string(&tt.to_string()))), (@ quote_span(proc_macro_crate.clone(), tt.span())), - ))), - TokenTree::Literal(tt) => minimal_quote!(crate::TokenTree::Literal({ + )), &mut ts);) + } + TokenTree::Literal(tt) => { + minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({ let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string()))) .parse::() .unwrap() @@ -120,16 +130,22 @@ pub fn quote(stream: TokenStream) -> TokenStream { } else { unreachable!() } - })) - })),)) - }) - .collect::(); - + }), &mut ts);) + } + } + .to_tokens(&mut tokens); + } if after_dollar { panic!("unexpected trailing `$` in `quote!`"); } - minimal_quote!([(@ tokens)].iter().cloned().collect::()) + minimal_quote! { + { + let mut ts = crate::TokenStream::new(); + (@ tokens) + ts + } + } } /// Quote a `Span` into a `TokenStream`. diff --git a/src/tools/clippy/tests/ui/auxiliary/proc_macro_attr.rs b/src/tools/clippy/tests/ui/auxiliary/proc_macro_attr.rs index e72d6b6ceadfd..077f6e961b469 100644 --- a/src/tools/clippy/tests/ui/auxiliary/proc_macro_attr.rs +++ b/src/tools/clippy/tests/ui/auxiliary/proc_macro_attr.rs @@ -1,4 +1,4 @@ -#![feature(repr128, proc_macro_hygiene, proc_macro_quote, box_patterns)] +#![feature(repr128, proc_macro_hygiene, proc_macro_quote, proc_macro_totokens, box_patterns)] #![allow(incomplete_features)] #![allow(clippy::useless_conversion, clippy::uninlined_format_args)] diff --git a/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs b/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs index fbf84337382e1..75040ebec523c 100644 --- a/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs +++ b/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs @@ -1,4 +1,4 @@ -#![feature(repr128, proc_macro_quote, proc_macro_span)] +#![feature(repr128, proc_macro_quote, proc_macro_totokens, proc_macro_span)] #![allow(incomplete_features)] #![allow(clippy::field_reassign_with_default)] #![allow(clippy::eq_op)] diff --git a/tests/ui/deriving/auxiliary/another-proc-macro.rs b/tests/ui/deriving/auxiliary/another-proc-macro.rs index 47f3c5b9c4b05..079b5a6787420 100644 --- a/tests/ui/deriving/auxiliary/another-proc-macro.rs +++ b/tests/ui/deriving/auxiliary/another-proc-macro.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; diff --git a/tests/ui/hygiene/auxiliary/opaque-hygiene.rs b/tests/ui/hygiene/auxiliary/opaque-hygiene.rs index 08dc592925aa5..1a8c21cee2d5e 100644 --- a/tests/ui/hygiene/auxiliary/opaque-hygiene.rs +++ b/tests/ui/hygiene/auxiliary/opaque-hygiene.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; use proc_macro::{TokenStream, quote}; diff --git a/tests/ui/macros/auxiliary/hello_macro.rs b/tests/ui/macros/auxiliary/hello_macro.rs index 79125a1f86e3a..a4f4ccbfbaefc 100644 --- a/tests/ui/macros/auxiliary/hello_macro.rs +++ b/tests/ui/macros/auxiliary/hello_macro.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; diff --git a/tests/ui/macros/auxiliary/issue-100199.rs b/tests/ui/macros/auxiliary/issue-100199.rs index f05c4f0722cfb..bfa9d5a9e889e 100644 --- a/tests/ui/macros/auxiliary/issue-100199.rs +++ b/tests/ui/macros/auxiliary/issue-100199.rs @@ -1,8 +1,9 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; -use proc_macro::{quote, Ident, Span, TokenStream, TokenTree}; +use proc_macro::{Ident, Span, TokenStream, TokenTree, quote}; #[proc_macro_attribute] pub fn struct_with_bound(_: TokenStream, _: TokenStream) -> TokenStream { diff --git a/tests/ui/macros/auxiliary/proc_macro_def.rs b/tests/ui/macros/auxiliary/proc_macro_def.rs index 38a1f6fa3c13b..34833cc1bd05b 100644 --- a/tests/ui/macros/auxiliary/proc_macro_def.rs +++ b/tests/ui/macros/auxiliary/proc_macro_def.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; diff --git a/tests/ui/macros/auxiliary/proc_macro_sequence.rs b/tests/ui/macros/auxiliary/proc_macro_sequence.rs index 0f5435401711e..dcbcc0a5f1fa0 100644 --- a/tests/ui/macros/auxiliary/proc_macro_sequence.rs +++ b/tests/ui/macros/auxiliary/proc_macro_sequence.rs @@ -1,8 +1,8 @@ -#![feature(proc_macro_span, proc_macro_quote)] +#![feature(proc_macro_span, proc_macro_quote, proc_macro_totokens)] extern crate proc_macro; -use proc_macro::{quote, Span, TokenStream, TokenTree}; +use proc_macro::{Span, TokenStream, TokenTree, quote}; // This macro generates a macro with the same macro definition as `manual_foo` in // `same-sequence-span.rs` but with the same span for all sequences. diff --git a/tests/ui/proc-macro/auxiliary/cond_plugin.rs b/tests/ui/proc-macro/auxiliary/cond_plugin.rs index 9858be230c95e..2b8dc7ed44ead 100644 --- a/tests/ui/proc-macro/auxiliary/cond_plugin.rs +++ b/tests/ui/proc-macro/auxiliary/cond_plugin.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; diff --git a/tests/ui/proc-macro/auxiliary/count_compound_ops.rs b/tests/ui/proc-macro/auxiliary/count_compound_ops.rs index c5a1c561976eb..9f7edfd8d3309 100644 --- a/tests/ui/proc-macro/auxiliary/count_compound_ops.rs +++ b/tests/ui/proc-macro/auxiliary/count_compound_ops.rs @@ -1,8 +1,9 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; -use proc_macro::{TokenStream, TokenTree, Spacing, Literal, quote}; +use proc_macro::{Literal, Spacing, TokenStream, TokenTree, quote}; #[proc_macro] pub fn count_compound_ops(input: TokenStream) -> TokenStream { diff --git a/tests/ui/proc-macro/auxiliary/double.rs b/tests/ui/proc-macro/auxiliary/double.rs index b6d952437ea1f..b2d447f67939e 100644 --- a/tests/ui/proc-macro/auxiliary/double.rs +++ b/tests/ui/proc-macro/auxiliary/double.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; use proc_macro::*; diff --git a/tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs b/tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs index 28d35c82148e6..5eb4c0e1d8ddd 100644 --- a/tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs +++ b/tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; use proc_macro::*; diff --git a/tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs b/tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs index dde997bf3ea3d..34c51cc9c246f 100644 --- a/tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs +++ b/tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro as proc_macro_renamed; // This does not break `quote!` diff --git a/tests/ui/proc-macro/auxiliary/mixed-site-span.rs b/tests/ui/proc-macro/auxiliary/mixed-site-span.rs index d837c88c9556a..5835c88ab098c 100644 --- a/tests/ui/proc-macro/auxiliary/mixed-site-span.rs +++ b/tests/ui/proc-macro/auxiliary/mixed-site-span.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; use proc_macro::*; @@ -13,10 +14,8 @@ pub fn proc_macro_rules(input: TokenStream) -> TokenStream { let local_use = id("local_use"); let mut single_quote = Punct::new('\'', Spacing::Joint); single_quote.set_span(Span::mixed_site()); - let label_use: TokenStream = [ - TokenTree::from(single_quote), - id("label_use"), - ].iter().cloned().collect(); + let label_use: TokenStream = + [TokenTree::from(single_quote), id("label_use")].iter().cloned().collect(); quote!( struct $item_def; let $local_def = 0; diff --git a/tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs b/tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs index 96aed8625aa5a..cca625cb6b131 100644 --- a/tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs +++ b/tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs @@ -1,4 +1,5 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; use proc_macro::{TokenStream, quote}; diff --git a/tests/ui/proc-macro/auxiliary/resolved-located-at.rs b/tests/ui/proc-macro/auxiliary/resolved-located-at.rs index 493956c00e987..6d9198edda35a 100644 --- a/tests/ui/proc-macro/auxiliary/resolved-located-at.rs +++ b/tests/ui/proc-macro/auxiliary/resolved-located-at.rs @@ -1,6 +1,7 @@ #![feature(proc_macro_def_site)] #![feature(proc_macro_diagnostic)] #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; use proc_macro::*; diff --git a/tests/ui/proc-macro/auxiliary/span-from-proc-macro.rs b/tests/ui/proc-macro/auxiliary/span-from-proc-macro.rs index 16ca5e3f9e2d1..8947ec9ce20d1 100644 --- a/tests/ui/proc-macro/auxiliary/span-from-proc-macro.rs +++ b/tests/ui/proc-macro/auxiliary/span-from-proc-macro.rs @@ -1,17 +1,18 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] #![feature(proc_macro_internals)] // FIXME - this shouldn't be necessary -extern crate proc_macro; extern crate custom_quote; +extern crate proc_macro; -use proc_macro::{quote, TokenStream}; +use proc_macro::{TokenStream, quote}; macro_rules! expand_to_quote { () => { quote! { let bang_error: bool = 25; } - } + }; } #[proc_macro] diff --git a/tests/ui/proc-macro/quote-debug.stdout b/tests/ui/proc-macro/quote-debug.stdout index d84b4e051e872..583599ff97524 100644 --- a/tests/ui/proc-macro/quote-debug.stdout +++ b/tests/ui/proc-macro/quote-debug.stdout @@ -19,25 +19,29 @@ extern crate std; extern crate proc_macro; fn main() { - [crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("let", - crate::Span::recover_proc_macro_span(0)))), - crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("hello", - crate::Span::recover_proc_macro_span(1)))), - crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new('=', - crate::Spacing::Alone))), - crate::TokenStream::from(crate::TokenTree::Literal({ - let mut iter = - "\"world\"".parse::().unwrap().into_iter(); - if let (Some(crate::TokenTree::Literal(mut lit)), None) = - (iter.next(), iter.next()) { - lit.set_span(crate::Span::recover_proc_macro_span(2)); - lit - } else { - ::core::panicking::panic("internal error: entered unreachable code") - } - })), - crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new(';', - crate::Spacing::Alone)))].iter().cloned().collect::() + { + let mut ts = crate::TokenStream::new(); + crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("let", + crate::Span::recover_proc_macro_span(0))), &mut ts); + crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("hello", + crate::Span::recover_proc_macro_span(1))), &mut ts); + crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new('=', + crate::Spacing::Alone)), &mut ts); + crate::ToTokens::to_tokens(&crate::TokenTree::Literal({ + let mut iter = + "\"world\"".parse::().unwrap().into_iter(); + if let (Some(crate::TokenTree::Literal(mut lit)), None) = + (iter.next(), iter.next()) { + lit.set_span(crate::Span::recover_proc_macro_span(2)); + lit + } else { + ::core::panicking::panic("internal error: entered unreachable code") + } + }), &mut ts); + crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(';', + crate::Spacing::Alone)), &mut ts); + ts + } } const _: () = { diff --git a/tests/ui/proc-macro/quote-interpolation.rs b/tests/ui/proc-macro/quote-interpolation.rs new file mode 100644 index 0000000000000..708e7135feecc --- /dev/null +++ b/tests/ui/proc-macro/quote-interpolation.rs @@ -0,0 +1,16 @@ +//@ check-pass + +#![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::*; + +fn main() { + let x = Ident::new("foo", Span::call_site()); + let _ = quote! { + let $x = 199; + }; +} diff --git a/tests/ui/proc-macro/span-from-proc-macro.stderr b/tests/ui/proc-macro/span-from-proc-macro.stderr index 452c04df8779e..b1365f647c4d0 100644 --- a/tests/ui/proc-macro/span-from-proc-macro.stderr +++ b/tests/ui/proc-macro/span-from-proc-macro.stderr @@ -1,5 +1,5 @@ error[E0412]: cannot find type `MissingType` in this scope - --> $DIR/auxiliary/span-from-proc-macro.rs:33:20 + --> $DIR/auxiliary/span-from-proc-macro.rs:34:20 | LL | pub fn error_from_attribute(_args: TokenStream, _input: TokenStream) -> TokenStream { | ----------------------------------------------------------------------------------- in this expansion of `#[error_from_attribute]` @@ -13,7 +13,7 @@ LL | #[error_from_attribute] | ----------------------- in this procedural macro expansion error[E0412]: cannot find type `OtherMissingType` in this scope - --> $DIR/auxiliary/span-from-proc-macro.rs:42:21 + --> $DIR/auxiliary/span-from-proc-macro.rs:43:21 | LL | pub fn error_from_derive(_input: TokenStream) -> TokenStream { | ------------------------------------------------------------ in this expansion of `#[derive(ErrorFromDerive)]` @@ -27,7 +27,7 @@ LL | #[derive(ErrorFromDerive)] | --------------- in this derive macro expansion error[E0425]: cannot find value `my_ident` in this scope - --> $DIR/auxiliary/span-from-proc-macro.rs:25:9 + --> $DIR/auxiliary/span-from-proc-macro.rs:26:9 | LL | pub fn other_error_from_bang(_input: TokenStream) -> TokenStream { | ---------------------------------------------------------------- in this expansion of `other_error_from_bang!` @@ -41,7 +41,7 @@ LL | other_error_from_bang!(); | ------------------------ in this macro invocation error[E0308]: mismatched types - --> $DIR/auxiliary/span-from-proc-macro.rs:12:36 + --> $DIR/auxiliary/span-from-proc-macro.rs:13:36 | LL | let bang_error: bool = 25; | ---- ^^ expected `bool`, found integer diff --git a/tests/ui/suggestions/auxiliary/proc-macro-type-error.rs b/tests/ui/suggestions/auxiliary/proc-macro-type-error.rs index 2930b87d1bbec..332d3ad80dd73 100644 --- a/tests/ui/suggestions/auxiliary/proc-macro-type-error.rs +++ b/tests/ui/suggestions/auxiliary/proc-macro-type-error.rs @@ -1,8 +1,9 @@ #![feature(proc_macro_quote)] +#![feature(proc_macro_totokens)] extern crate proc_macro; -use proc_macro::{quote, TokenStream}; +use proc_macro::{TokenStream, quote}; #[proc_macro_attribute] pub fn hello(_: TokenStream, _: TokenStream) -> TokenStream {