diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index bdaa865a998d6..40ca1e221b15b 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -2,9 +2,13 @@ use std::cell::RefCell; use std::marker::PhantomData; +use std::num::NonZero; use std::sync::atomic::AtomicU32; use super::*; +use crate::StandaloneLevel; +use crate::bridge::server::{Dispatcher, DispatcherTrait}; +use crate::bridge::standalone::NoRustc; macro_rules! define_client_handles { ( @@ -107,6 +111,10 @@ impl Clone for TokenStream { } impl Span { + pub(crate) fn dummy() -> Span { + Span { handle: NonZero::new(1).unwrap() } + } + pub(crate) fn def_site() -> Span { Bridge::with(|bridge| bridge.globals.def_site) } @@ -141,7 +149,10 @@ macro_rules! define_client_side { api_tags::Method::$name(api_tags::$name::$method).encode(&mut buf, &mut ()); $($arg.encode(&mut buf, &mut ());)* - buf = bridge.dispatch.call(buf); + buf = match &mut bridge.dispatch { + DispatchWay::Closure(f) => f.call(buf), + DispatchWay::Directly(disp) => disp.dispatch(buf), + }; let r = Result::<_, PanicMessage>::decode(&mut &buf[..], &mut ()); @@ -155,13 +166,18 @@ macro_rules! define_client_side { } with_api!(self, self, define_client_side); +enum DispatchWay<'a> { + Closure(closure::Closure<'a, Buffer, Buffer>), + Directly(Dispatcher), +} + struct Bridge<'a> { /// Reusable buffer (only `clear`-ed, never shrunk), primarily /// used for making requests. cached_buffer: Buffer, /// Server-side function that the client uses to make requests. - dispatch: closure::Closure<'a, Buffer, Buffer>, + dispatch: DispatchWay<'a>, /// Provided globals for this macro expansion. globals: ExpnGlobals, @@ -173,12 +189,33 @@ impl<'a> !Sync for Bridge<'a> {} #[allow(unsafe_code)] mod state { use std::cell::{Cell, RefCell}; + use std::marker::PhantomData; use std::ptr; use super::Bridge; + use crate::StandaloneLevel; + use crate::bridge::buffer::Buffer; + use crate::bridge::client::{COUNTERS, DispatchWay}; + use crate::bridge::server::{Dispatcher, HandleStore, MarkedTypes}; + use crate::bridge::{ExpnGlobals, Marked, standalone}; thread_local! { static BRIDGE_STATE: Cell<*const ()> = const { Cell::new(ptr::null()) }; + static STANDALONE: RefCell> = RefCell::new(standalone_bridge()); + pub(super) static USE_STANDALONE: Cell = const { Cell::new(StandaloneLevel::Never) }; + } + + fn standalone_bridge() -> Bridge<'static> { + let mut store = HandleStore::new(&COUNTERS); + let id = store.Span.alloc(Marked { value: standalone::Span, _marker: PhantomData }); + let dummy = super::Span { handle: id }; + let dispatcher = + Dispatcher { handle_store: store, server: MarkedTypes(standalone::NoRustc) }; + Bridge { + cached_buffer: Buffer::new(), + dispatch: DispatchWay::Directly(dispatcher), + globals: ExpnGlobals { call_site: dummy, def_site: dummy, mixed_site: dummy }, + } } pub(super) fn set<'bridge, R>(state: &RefCell>, f: impl FnOnce() -> R) -> R { @@ -199,16 +236,23 @@ mod state { pub(super) fn with( f: impl for<'bridge> FnOnce(Option<&RefCell>>) -> R, ) -> R { - let state = BRIDGE_STATE.get(); - // SAFETY: the only place where the pointer is set is in `set`. It puts - // back the previous value after the inner call has returned, so we know - // that as long as the pointer is not null, it came from a reference to - // a `RefCell` that outlasts the call to this function. Since `f` - // works the same for any lifetime of the bridge, including the actual - // one, we can lie here and say that the lifetime is `'static` without - // anyone noticing. - let bridge = unsafe { state.cast::>>().as_ref() }; - f(bridge) + let level = USE_STANDALONE.get(); + if level == StandaloneLevel::Always + || (level == StandaloneLevel::FallbackOnly && BRIDGE_STATE.get().is_null()) + { + STANDALONE.with(|bridge| f(Some(bridge))) + } else { + let state = BRIDGE_STATE.get(); + // SAFETY: the only place where the pointer is set is in `set`. It puts + // back the previous value after the inner call has returned, so we know + // that as long as the pointer is not null, it came from a reference to + // a `RefCell` that outlasts the call to this function. Since `f` + // works the same for any lifetime of the bridge, including the actual + // one, we can lie here and say that the lifetime is `'static` without + // anyone noticing. + let bridge = unsafe { state.cast::>>().as_ref() }; + f(bridge) + } } } @@ -228,6 +272,10 @@ pub(crate) fn is_available() -> bool { state::with(|s| s.is_some()) } +pub(crate) fn enable_standalone(level: StandaloneLevel) { + state::USE_STANDALONE.set(level); +} + /// A client-side RPC entry-point, which may be using a different `proc_macro` /// from the one used by the server, but can be invoked compatibly. /// @@ -292,7 +340,11 @@ fn run_client Decode<'a, 's, ()>, R: Encode<()>>( let (globals, input) = <(ExpnGlobals, A)>::decode(reader, &mut ()); // Put the buffer we used for input back in the `Bridge` for requests. - let state = RefCell::new(Bridge { cached_buffer: buf.take(), dispatch, globals }); + let state = RefCell::new(Bridge { + cached_buffer: buf.take(), + dispatch: DispatchWay::Closure(dispatch), + globals, + }); let output = state::set(&state, || f(input)); diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index b0ee9c0cc3027..4f5fe2a08b8b7 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -138,6 +138,7 @@ mod rpc; mod selfless_reify; #[forbid(unsafe_code)] pub mod server; +pub(crate) mod standalone; #[allow(unsafe_code)] mod symbol; diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index e9ef26c07f24f..051151b6c7c69 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -12,12 +12,12 @@ macro_rules! define_server_handles { ) => { #[allow(non_snake_case)] pub(super) struct HandleStore { - $($oty: handle::OwnedStore,)* - $($ity: handle::InternedStore,)* + $(pub(super) $oty: handle::OwnedStore,)* + $(pub(super) $ity: handle::InternedStore,)* } impl HandleStore { - fn new(handle_counters: &'static client::HandleCounters) -> Self { + pub(super) fn new(handle_counters: &'static client::HandleCounters) -> Self { HandleStore { $($oty: handle::OwnedStore::new(&handle_counters.$oty),)* $($ity: handle::InternedStore::new(&handle_counters.$ity),)* @@ -119,7 +119,7 @@ macro_rules! declare_server_traits { } with_api!(Self, self_, declare_server_traits); -pub(super) struct MarkedTypes(S); +pub(super) struct MarkedTypes(pub(super) S); impl Server for MarkedTypes { fn globals(&mut self) -> ExpnGlobals { @@ -150,9 +150,9 @@ macro_rules! define_mark_types_impls { } with_api!(Self, self_, define_mark_types_impls); -struct Dispatcher { - handle_store: HandleStore, - server: S, +pub(super) struct Dispatcher { + pub(super) handle_store: HandleStore>, + pub(super) server: MarkedTypes, } macro_rules! define_dispatcher_impl { @@ -167,7 +167,7 @@ macro_rules! define_dispatcher_impl { fn dispatch(&mut self, buf: Buffer) -> Buffer; } - impl DispatcherTrait for Dispatcher> { + impl DispatcherTrait for Dispatcher { $(type $name = as Types>::$name;)* fn dispatch(&mut self, mut buf: Buffer) -> Buffer { diff --git a/library/proc_macro/src/bridge/standalone.rs b/library/proc_macro/src/bridge/standalone.rs new file mode 100644 index 0000000000000..1ee204c82cefa --- /dev/null +++ b/library/proc_macro/src/bridge/standalone.rs @@ -0,0 +1,506 @@ +#![warn(warnings)] +use std::cell::{Cell, RefCell}; +use std::ops::{Bound, Range, RangeBounds}; + +use crate::bridge::client::Symbol; +use crate::bridge::fxhash::FxHashMap; +use crate::bridge::{ + self, DelimSpan, Diagnostic, ExpnGlobals, Group, LitKind, Punct, TokenTree, server, +}; +use crate::{Delimiter, LEGAL_PUNCT_CHARS}; + +type Result = std::result::Result; +type Literal = bridge::Literal; + +pub struct NoRustc; + +impl server::Span for NoRustc { + fn debug(&mut self, _: Self::Span) -> String { + "Span".to_string() + } + + fn parent(&mut self, _: Self::Span) -> Option { + None + } + + fn source(&mut self, _: Self::Span) -> Self::Span { + Span + } + + fn byte_range(&mut self, _: Self::Span) -> Range { + 0..0 + } + + fn start(&mut self, _: Self::Span) -> Self::Span { + Span + } + + fn end(&mut self, _: Self::Span) -> Self::Span { + Span + } + + fn line(&mut self, _: Self::Span) -> usize { + 1 + } + + fn column(&mut self, _: Self::Span) -> usize { + 1 + } + + fn file(&mut self, _: Self::Span) -> String { + "".to_string() + } + + fn local_file(&mut self, _: Self::Span) -> Option { + None + } + + fn join(&mut self, _: Self::Span, _: Self::Span) -> Option { + Some(Span) + } + + fn subspan( + &mut self, + _: Self::Span, + _start: Bound, + _end: Bound, + ) -> Option { + Some(Span) + } + + fn resolved_at(&mut self, _span: Self::Span, _at: Self::Span) -> Self::Span { + Span + } + + fn source_text(&mut self, _: Self::Span) -> Option { + None + } + + fn save_span(&mut self, _: Self::Span) -> usize { + let n = SAVED_SPAN_COUNT.get(); + SAVED_SPAN_COUNT.set(n + 1); + n + } + + fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span { + if id < SAVED_SPAN_COUNT.get() { + Span + } else { + panic!("recovered span index out of bounds"); + } + } +} + +thread_local! { + static SAVED_SPAN_COUNT: Cell = const { Cell::new(0) }; + static TRACKED_ENV_VARS: RefCell>> = RefCell::new(FxHashMap::default()); +} + +fn parse_maybe_raw_str( + mut s: &str, + raw_variant: fn(u8) -> LitKind, + regular_variant: LitKind, +) -> Result { + /// Returns a string containing exactly `num` '#' characters. + /// Uses a 256-character source string literal which is always safe to + /// index with a `u8` index. + fn get_hashes_str(num: u8) -> &'static str { + const HASHES: &str = "\ + ################################################################\ + ################################################################\ + ################################################################\ + ################################################################\ + "; + const _: () = assert!(HASHES.len() == 256); + &HASHES[..num as usize] + } + let mut hash_count = None; + + if s.starts_with('r') { + s = s.strip_prefix('r').unwrap(); + let mut h = 0; + for c in s.chars() { + if c == '#' { + if h == u8::MAX { + return Err(()); + } + h += 1; + } else { + break; + } + } + hash_count = Some(h); + let hashes = get_hashes_str(h); + s = s.strip_prefix(hashes).unwrap(); + s = s.strip_suffix(hashes).ok_or(())?; + } + let sym = parse_plain_str(s)?; + + Ok(make_literal(if let Some(h) = hash_count { raw_variant(h) } else { regular_variant }, sym)) +} + +fn parse_char(s: &str) -> Result { + if s.chars().count() == 1 { Ok(make_literal(LitKind::Char, Symbol::new(s))) } else { Err(()) } +} + +fn parse_plain_str(mut s: &str) -> Result { + s = s.strip_prefix("\"").ok_or(())?.strip_suffix('\"').ok_or(())?; + Ok(Symbol::new(s)) +} + +const INT_SUFFIXES: &[&str] = + &["u8", "i8", "u16", "i16", "u32", "i32", "u64", "i64", "u128", "i128"]; +const FLOAT_SUFFIXES: &[&str] = &["f16", "f32", "f64", "f128"]; + +fn parse_numeral(mut s: &str) -> Result { + for suffix in INT_SUFFIXES { + if s.ends_with(suffix) { + return parse_integer(s); + } + } + let is_negative = s.starts_with('-'); + let non_negative = s.trim_prefix('-'); + if non_negative.starts_with("0b") + || non_negative.starts_with("0o") + || non_negative.starts_with("0x") + { + return parse_integer(s); + } + let (s, suffix) = strip_number_suffix(s, FLOAT_SUFFIXES); + + Ok(Literal { kind: LitKind::Float, symbol: Symbol::new(s), suffix, span: Span }) +} + +fn parse_integer(mut s: &str) -> Result { + let is_negative = s.starts_with('-'); + s = s.trim_prefix('-'); + + let (s, valid_chars) = if let Some(s) = s.strip_prefix("0b") { + (s, '0'..='1') + } else if let Some(s) = s.strip_prefix("0o") { + (s, '0'..='7') + } else if let Some(s) = s.strip_prefix("0x") { + (s, '0'..='F') + } else { + (s, '0'..='9') + }; + + let (s, suffix) = strip_number_suffix(s, INT_SUFFIXES); + + let mut any_found = false; + for c in s.chars() { + if c == '_' { + continue; + } + if valid_chars.contains(&c) { + any_found = true; + continue; + } + return Err(()); + } + if !any_found { + return Err(()); + } + + Ok(Literal { kind: LitKind::Integer, symbol: Symbol::new(s), suffix, span: Span }) +} + +fn strip_number_suffix<'a>(s: &'a str, suffixes: &[&str]) -> (&'a str, Option) { + for suf in suffixes { + if let Some(new_s) = s.strip_suffix(suf) { + return (new_s, Some(Symbol::new(suf))); + } + } + (s, None) +} + +fn make_literal(kind: LitKind, symbol: Symbol) -> Literal { + Literal { kind, symbol, suffix: None, span: Span } +} + +impl server::FreeFunctions for NoRustc { + fn injected_env_var(&mut self, var: &str) -> Option { + TRACKED_ENV_VARS.with_borrow(|vars| vars.get(var)?.clone()) + } + + fn track_env_var(&mut self, var: &str, value: Option<&str>) { + TRACKED_ENV_VARS + .with_borrow_mut(|vars| vars.insert(var.to_string(), value.map(ToString::to_string))); + } + + fn track_path(&mut self, _path: &str) {} + + fn literal_from_str(&mut self, s: &str) -> Result { + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return Err(()); + }; + let rest = &s[1..]; + + match first { + 'b' => { + if chars.next() == Some('\'') { + parse_char(rest).map(|mut lit| { + lit.kind = LitKind::Byte; + lit + }) + } else { + parse_maybe_raw_str(rest, LitKind::ByteStrRaw, LitKind::ByteStr) + } + } + 'c' => parse_maybe_raw_str(rest, LitKind::CStrRaw, LitKind::CStr), + 'r' => parse_maybe_raw_str(rest, LitKind::StrRaw, LitKind::Str), + '0'..='9' | '-' => parse_numeral(s), + '\'' => parse_char(s), + '"' => Ok(make_literal(LitKind::Str, parse_plain_str(s)?)), + _ => Err(()), + } + } + + fn emit_diagnostic(&mut self, _: Diagnostic) { + panic!("cannot emit diagnostic in standalone mode"); + } +} + +impl server::TokenStream for NoRustc { + fn is_empty(&mut self, tokens: &Self::TokenStream) -> bool { + tokens.0.is_empty() + } + + fn expand_expr(&mut self, _tokens: &Self::TokenStream) -> Result { + todo!("`expand_expr` is not yet supported in the standalone backend") + } + + fn from_str(&mut self, src: &str) -> Self::TokenStream { + return TokenStream::new(); + + /// Returns the delimiter, and whether it is the opening form. + fn char_to_delim(c: char) -> Option<(Delimiter, bool)> { + Some(match c { + '(' => (Delimiter::Parenthesis, true), + ')' => (Delimiter::Parenthesis, false), + '{' => (Delimiter::Brace, true), + '}' => (Delimiter::Brace, false), + '[' => (Delimiter::Bracket, true), + ']' => (Delimiter::Bracket, false), + _ => return None, + }) + } + + let mut unfinished_streams = vec![TokenStream::new()]; + let mut unclosed_delimiters = Vec::new(); + let mut current_ident = String::new(); + for c in src.chars() { + if let Some((delim, is_opening)) = char_to_delim(c) { + if is_opening { + unclosed_delimiters.push(delim); + unfinished_streams.push(TokenStream::new()); + } else if unclosed_delimiters.pop() == Some(delim) { + let group = TokenTree::<_, _, Symbol>::Group(Group { + delimiter: delim, + stream: unfinished_streams.pop(), + span: DelimSpan::from_single(Span), + }); + unfinished_streams.last_mut().unwrap().0.push(group); + } else { + panic!("cannot parse string into token stream") + } + } else if LEGAL_PUNCT_CHARS.contains(&c) { + unfinished_streams.last_mut().unwrap().0.push(TokenTree::Punct(Punct { + ch: c as u8, + joint: todo!(), + span: Span, + })); + } + match c { + _ => todo!(), + } + } + unfinished_streams[0].clone() + } + + fn to_string(&mut self, tokens: &Self::TokenStream) -> String { + /* + /// Returns a string containing exactly `num` '#' characters. + /// Uses a 256-character source string literal which is always safe to + /// index with a `u8` index. + fn get_hashes_str(num: u8) -> &'static str { + const HASHES: &str = "\ + ################################################################\ + ################################################################\ + ################################################################\ + ################################################################\ + "; + const _: () = assert!(HASHES.len() == 256); + &HASHES[..num as usize] + }*/ + + let mut s = String::new(); + let mut last = String::new(); + let mut second_last = String::new(); + + for (idx, tree) in tokens.0.iter().enumerate() { + let mut space = true; + let new_part = match tree { + TokenTree::Group(group) => { + let inner = if let Some(stream) = &group.stream { + self.to_string(stream) + } else { + String::new() + }; + match group.delimiter { + Delimiter::Parenthesis => format!("({inner})"), + Delimiter::Brace => { + if inner.is_empty() { + "{ }".to_string() + } else { + format!("{{ {inner} }}") + } + } + Delimiter::Bracket => format!("[{inner}]"), + Delimiter::None => inner, + } + } + TokenTree::Ident(ident) => { + if ident.is_raw { + format!("r#{}", ident.sym) + } else { + ident.sym.to_string() + } + } + TokenTree::Literal(lit) => { + let respanned = bridge::Literal { + kind: lit.kind, + symbol: lit.symbol, + suffix: lit.suffix, + span: super::client::Span::dummy(), + }; + crate::Literal(respanned).to_string() + /*let inner = if let Some(suffix) = lit.suffix { + format!("{}{suffix}", lit.symbol) + } else { + lit.symbol.to_string() + }; + match lit.kind { + LitKind::Byte => format!("b'{inner}'"), + LitKind::ByteStr => format!("b\"{inner}\""), + LitKind::ByteStrRaw(raw) => { + format!("br{0}\"{inner}\"{0}", get_hashes_str(raw)) + } + LitKind::CStr => format!("c\"{inner}\""), + LitKind::CStrRaw(raw) => { + format!("cr{0}\"{inner}\"{0}", get_hashes_str(raw)) + } + LitKind::Char => format!("'{inner}'"), + LitKind::ErrWithGuar => unreachable!(), + LitKind::Float | LitKind::Integer => inner, + LitKind::Str => format!("\"{inner}\""), + LitKind::StrRaw(raw) => format!("r{0}\"{inner}\"{0}", get_hashes_str(raw)), + }*/ + } + TokenTree::Punct(punct) => { + let c = punct.ch as char; + if c == '\'' { + space = false; + } + c.to_string() + } + }; + + const NON_SEPARATABLE_TOKENS: &[(char, char)] = &[(':', ':'), ('-', '>'), ('=', '>')]; + + for (first, second) in NON_SEPARATABLE_TOKENS { + if second_last == first.to_string() && last == second.to_string() && new_part != ":" + { + s.pop(); // pop ' ' + s.pop(); // pop `second` + s.pop(); // pop ' ' + s.push(*second); + s.push(' '); + } + } + s.push_str(&new_part); + second_last = last; + last = new_part; + if space && idx + 1 != tokens.0.len() { + s.push(' '); + } + } + s + } + + fn from_token_tree( + &mut self, + tree: TokenTree, + ) -> Self::TokenStream { + TokenStream(vec![tree]) + } + + fn concat_trees( + &mut self, + base: Option, + trees: Vec>, + ) -> Self::TokenStream { + let mut base = base.unwrap_or_else(TokenStream::new); + base.0.extend(trees); + base + } + + fn concat_streams( + &mut self, + base: Option, + streams: Vec, + ) -> Self::TokenStream { + let mut base = base.unwrap_or_else(TokenStream::new); + for stream in streams { + base = self.concat_trees(Some(base), stream.0); + } + base + } + + fn into_trees( + &mut self, + tokens: Self::TokenStream, + ) -> Vec> { + tokens.0 + } +} + +pub struct FreeFunctions; +#[derive(Clone, Default)] +pub struct TokenStream(Vec>); +impl TokenStream { + fn new() -> Self { + Self(Vec::new()) + } +} + +#[derive(Hash, PartialEq, Eq, Clone, Copy)] +pub struct Span; + +impl server::Types for NoRustc { + type FreeFunctions = FreeFunctions; + type TokenStream = TokenStream; + type Span = Span; + type Symbol = Symbol; +} + +impl server::Server for NoRustc { + fn globals(&mut self) -> ExpnGlobals { + ExpnGlobals { def_site: Span, call_site: Span, mixed_site: Span } + } + + fn intern_symbol(ident: &str) -> Self::Symbol { + Symbol::new(ident) + } + + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + symbol.with(f); + } +} + +impl server::Symbol for NoRustc { + fn normalize_and_validate_ident(&mut self, string: &str) -> Result { + todo!() + } +} diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 4efdfcad924b5..e97cbdecbf89d 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -28,6 +28,7 @@ #![feature(restricted_std)] #![feature(rustc_attrs)] #![feature(extend_one)] +#![feature(trim_prefix_suffix)] #![recursion_limit = "256"] #![allow(internal_features)] #![deny(ffi_unwind_calls)] @@ -89,6 +90,33 @@ pub fn is_available() -> bool { bridge::client::is_available() } +/// Controls the extent to which the new standalone backend is used. +/// +/// When this will be stabilized, the default level will change from +/// `Never` to `FallbackOnly`. +#[unstable(feature = "proc_macro_standalone", issue = "130856")] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum StandaloneLevel { + /// The standalone implementation is never used. This is the default. + Never, + /// The standalone implementation is only used outside of procedural macros. + FallbackOnly, + /// The standalone implementation is always used, even in procedural macros. + /// + /// This does not actually work and should be removed before merging. + Always, +} + +/// Enables the new experimental standalone backend, which allows calling the +/// functions in this crate outside of procedural macros. +/// +/// When stabilizing this feature, this function will be removed and all programs +/// will have the fallback activated automatically. +#[unstable(feature = "proc_macro_standalone", issue = "130856")] +pub fn enable_standalone() { + bridge::client::enable_standalone(StandaloneLevel::FallbackOnly); +} + /// The main type provided by this crate, representing an abstract stream of /// tokens, or, more specifically, a sequence of token trees. /// The type provides interfaces for iterating over those token trees and, conversely, @@ -947,6 +975,11 @@ pub enum Spacing { Alone, } +pub(crate) const LEGAL_PUNCT_CHARS: &[char] = &[ + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', ':', '#', + '$', '?', '\'', +]; + impl Punct { /// Creates a new `Punct` from the given character and spacing. /// The `ch` argument must be a valid punctuation character permitted by the language, @@ -956,11 +989,7 @@ impl Punct { /// which can be further configured with the `set_span` method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(ch: char, spacing: Spacing) -> Punct { - const LEGAL_CHARS: &[char] = &[ - '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', - ':', '#', '$', '?', '\'', - ]; - if !LEGAL_CHARS.contains(&ch) { + if !LEGAL_PUNCT_CHARS.contains(&ch) { panic!("unsupported character `{:?}`", ch); } Punct(bridge::Punct { @@ -1156,7 +1185,7 @@ macro_rules! unsuffixed_int_literals { /// specified on this token, meaning that invocations like /// `Literal::i8_unsuffixed(1)` are equivalent to /// `Literal::u32_unsuffixed(1)`. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// Literals created through this method have the `Span::call_site()` @@ -1219,7 +1248,7 @@ impl Literal { /// This constructor is similar to those like `Literal::i8_unsuffixed` where /// the float's value is emitted directly into the token but no suffix is /// used, so it may be inferred to be a `f64` later in the compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics @@ -1244,7 +1273,7 @@ impl Literal { /// specified is the preceding part of the token and `f32` is the suffix of /// the token. This token will always be inferred to be an `f32` in the /// compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics @@ -1264,7 +1293,7 @@ impl Literal { /// This constructor is similar to those like `Literal::i8_unsuffixed` where /// the float's value is emitted directly into the token but no suffix is /// used, so it may be inferred to be a `f64` later in the compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics @@ -1289,7 +1318,7 @@ impl Literal { /// specified is the preceding part of the token and `f64` is the suffix of /// the token. This token will always be inferred to be an `f64` in the /// compiler. - /// Literals created from negative numbers might not survive rountrips through + /// Literals created from negative numbers might not survive roundtrips through /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal). /// /// # Panics diff --git a/tests/ui/proc-macro/auxiliary/nonfatal-parsing-body.rs b/tests/ui/proc-macro/auxiliary/nonfatal-parsing-body.rs new file mode 100644 index 0000000000000..515fa8f7204c2 --- /dev/null +++ b/tests/ui/proc-macro/auxiliary/nonfatal-parsing-body.rs @@ -0,0 +1,104 @@ +use proc_macro::*; +use std::panic::catch_unwind; +use std::str::FromStr; + +fn stream(s: &str) { + println!("{:?}", TokenStream::from_str(s)); +} + +fn lit(s: &str) { + println!("{:?}", Literal::from_str(s)); +} + +fn stream_catch_unwind(s: &str) { + if catch_unwind(|| println!("{:?}", TokenStream::from_str(s))).is_ok() { + eprintln!("{s} did not panic"); + } +} + +fn lit_catch_unwind(s: &str) { + if catch_unwind(|| println!("{:?}", Literal::from_str(s))).is_ok() { + eprintln!("{s} did not panic"); + } +} + +pub fn run() { + // returns Ok(valid instance) + lit("123"); + lit("\"ab\""); + lit("\'b\'"); + lit("'b'"); + lit("b\"b\""); + lit("c\"b\""); + lit("cr\"b\""); + lit("b'b'"); + lit("256u8"); + lit("-256u8"); + stream("-256u8"); + lit("0b11111000000001111i16"); + lit("0xf32"); + lit("0b0f32"); + lit("2E4"); + lit("2.2E-4f64"); + lit("18u8E"); + lit("18.0u8E"); + lit("cr#\"// /* // \n */\"#"); + lit("'\\''"); + lit("'\\\''"); + lit(&format!("r{0}\"a\"{0}", "#".repeat(255))); + stream("fn main() { println!(\"Hello, world!\") }"); + stream("18.u8E"); + stream("18.0f32"); + stream("18.0f34"); + stream("18.bu8"); + stream("3//\n4"); + stream( + "\'c\'/*\n + */", + ); + stream("/*a*/ //"); + + println!("### ERRORS"); + + // returns Err(LexError) + lit("\'c\'/**/"); + lit(" 0"); + lit("0 "); + lit("0//"); + lit("3//\n4"); + lit("18.u8E"); + lit("/*a*/ //"); + // FIXME: all of the cases below should return an Err and emit no diagnostics, but don't yet. + + // emits diagnostics and returns LexError + lit("r'r'"); + lit("c'r'"); + + // emits diagnostic and returns a seemingly valid tokenstream + stream("r'r'"); + stream("c'r'"); + + for (parse, parse_catch_unwind) in [ + (stream as fn(&str), stream_catch_unwind as fn(&str)), + (lit, lit_catch_unwind), + ] { + // emits diagnostic(s), then panics + parse_catch_unwind("1 ) 2"); + parse_catch_unwind("( x [ ) ]"); + parse_catch_unwind("r#"); + + // emits diagnostic(s), then returns Ok(Literal { kind: ErrWithGuar, .. }) + parse("0b2"); + parse("0bf32"); + parse("0b0.0f32"); + parse("'\''"); + parse( + "' +'", + ); + parse_catch_unwind(&format!("r{0}\"a\"{0}", "#".repeat(256))); + + // emits diagnostic, then, when parsing as a lit, returns LexError, otherwise ErrWithGuar + parse("/*a*/ 0b2 //"); + } +} diff --git a/tests/ui/proc-macro/auxiliary/nonfatal-parsing.rs b/tests/ui/proc-macro/auxiliary/nonfatal-parsing.rs new file mode 100644 index 0000000000000..75db8ee3c5fc8 --- /dev/null +++ b/tests/ui/proc-macro/auxiliary/nonfatal-parsing.rs @@ -0,0 +1,11 @@ +extern crate proc_macro; +use proc_macro::*; + +#[path = "nonfatal-parsing-body.rs"] +mod body; + +#[proc_macro] +pub fn run(_: TokenStream) -> TokenStream { + body::run(); + TokenStream::new() +} diff --git a/tests/ui/proc-macro/nonfatal-parsing.in_macro.stderr b/tests/ui/proc-macro/nonfatal-parsing.in_macro.stderr new file mode 100644 index 0000000000000..805bbc86d0caa --- /dev/null +++ b/tests/ui/proc-macro/nonfatal-parsing.in_macro.stderr @@ -0,0 +1,197 @@ +error: prefix `r` is unknown + --> :1:1 + | +LL | r'r' + | ^ unknown prefix + | + = note: prefixed identifiers and literals are reserved since Rust 2021 +help: consider inserting whitespace here + | +LL | r 'r' + | + + +error: prefix `c` is unknown + --> :1:1 + | +LL | c'r' + | ^ unknown prefix + | + = note: prefixed identifiers and literals are reserved since Rust 2021 +help: consider inserting whitespace here + | +LL | c 'r' + | + + +error: unexpected closing delimiter: `)` + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ unexpected closing delimiter + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: unexpected closing delimiter: `]` + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | -^^^^^^^^^^^^^^^^^^^^^^^ + | | + | the nearest open delimiter + | missing open `(` for this delimiter + | unexpected closing delimiter + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: found invalid character; only `#` is allowed in raw string delimitation: \u{0} + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: invalid digit for a base 2 literal + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0768]: no valid digits found for number + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: binary float literal is not supported + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: character constant must be escaped: `'` + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) +help: escape the character + | +LL - nonfatal_parsing::run!(); +LL + nonfatal_parsing::run!(\'; + | + +error: character constant must be escaped: `\n` + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) +help: escape the character + | +LL - nonfatal_parsing::run!(); +LL + nonfatal_parsing::run!(\n; + | + +error: too many `#` symbols: raw strings may be delimited by up to 255 `#` symbols, but found 256 + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: invalid digit for a base 2 literal + --> $DIR/nonfatal-parsing.rs:20:5 + | +LL | nonfatal_parsing::run!(); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no` + = note: this error originates in the macro `nonfatal_parsing::run` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: unexpected closing delimiter: `)` + --> :1:3 + | +LL | 1 ) 2 + | ^ unexpected closing delimiter + +error: unexpected closing delimiter: `]` + --> :1:10 + | +LL | ( x [ ) ] + | - - ^ unexpected closing delimiter + | | | + | | missing open `(` for this delimiter + | the nearest open delimiter + +error: found invalid character; only `#` is allowed in raw string delimitation: \u{0} + --> :1:1 + | +LL | r# + | ^^ + +error: invalid digit for a base 2 literal + --> :1:3 + | +LL | 0b2 + | ^ + +error[E0768]: no valid digits found for number + --> :1:1 + | +LL | 0bf32 + | ^^ + +error: binary float literal is not supported + --> :1:1 + | +LL | 0b0.0f32 + | ^^^^^ + +error: character constant must be escaped: `'` + --> :1:2 + | +LL | ''' + | ^ + | +help: escape the character + | +LL | '\'' + | + + +error: character constant must be escaped: `\n` + --> :1:2 + | +LL | ' + | __^ +LL | | ' + | |_^ + | +help: escape the character + | +LL | '\n' + | ++ + +error: too many `#` symbols: raw strings may be delimited by up to 255 `#` symbols, but found 256 + --> :1:1 + | +LL | r#######################################...################################################## + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^...^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: invalid digit for a base 2 literal + --> :1:9 + | +LL | /*a*/ 0b2 // + | ^ + +error: aborting due to 22 previous errors + +For more information about this error, try `rustc --explain E0768`. diff --git a/tests/ui/proc-macro/nonfatal-parsing.in_macro.stdout b/tests/ui/proc-macro/nonfatal-parsing.in_macro.stdout new file mode 100644 index 0000000000000..c3d6345dfa26f --- /dev/null +++ b/tests/ui/proc-macro/nonfatal-parsing.in_macro.stdout @@ -0,0 +1,54 @@ +Ok(Literal { kind: Integer, symbol: "123", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Str, symbol: "ab", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Char, symbol: "b", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Char, symbol: "b", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: ByteStr, symbol: "b", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: CStr, symbol: "b", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: CStrRaw(0), symbol: "b", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Byte, symbol: "b", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Integer, symbol: "256", suffix: Some("u8"), span: #25 bytes(460..484) }) +Ok(Literal { kind: Integer, symbol: "-256", suffix: Some("u8"), span: #25 bytes(460..484) }) +Ok(TokenStream [Punct { ch: '-', spacing: Alone, span: #25 bytes(460..484) }, Literal { kind: Integer, symbol: "256", suffix: Some("u8"), span: #25 bytes(460..484) }]) +Ok(Literal { kind: Integer, symbol: "0b11111000000001111", suffix: Some("i16"), span: #25 bytes(460..484) }) +Ok(Literal { kind: Integer, symbol: "0xf32", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Integer, symbol: "0b0", suffix: Some("f32"), span: #25 bytes(460..484) }) +Ok(Literal { kind: Float, symbol: "2E4", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Float, symbol: "2.2E-4", suffix: Some("f64"), span: #25 bytes(460..484) }) +Ok(Literal { kind: Integer, symbol: "18", suffix: Some("u8E"), span: #25 bytes(460..484) }) +Ok(Literal { kind: Float, symbol: "18.0", suffix: Some("u8E"), span: #25 bytes(460..484) }) +Ok(Literal { kind: CStrRaw(1), symbol: "// /* // \n */", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Char, symbol: "\'", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: Char, symbol: "\'", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: StrRaw(255), symbol: "a", suffix: None, span: #25 bytes(460..484) }) +Ok(TokenStream [Ident { ident: "fn", span: #25 bytes(460..484) }, Ident { ident: "main", span: #25 bytes(460..484) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #25 bytes(460..484) }, Group { delimiter: Brace, stream: TokenStream [Ident { ident: "println", span: #25 bytes(460..484) }, Punct { ch: '!', spacing: Alone, span: #25 bytes(460..484) }, Group { delimiter: Parenthesis, stream: TokenStream [Literal { kind: Str, symbol: "Hello, world!", suffix: None, span: #25 bytes(460..484) }], span: #25 bytes(460..484) }], span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: Integer, symbol: "18", suffix: None, span: #25 bytes(460..484) }, Punct { ch: '.', spacing: Alone, span: #25 bytes(460..484) }, Ident { ident: "u8E", span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: Float, symbol: "18.0", suffix: Some("f32"), span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: Float, symbol: "18.0", suffix: Some("f34"), span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: Integer, symbol: "18", suffix: None, span: #25 bytes(460..484) }, Punct { ch: '.', spacing: Alone, span: #25 bytes(460..484) }, Ident { ident: "bu8", span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: Integer, symbol: "3", suffix: None, span: #25 bytes(460..484) }, Literal { kind: Integer, symbol: "4", suffix: None, span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: Char, symbol: "c", suffix: None, span: #25 bytes(460..484) }]) +Ok(TokenStream []) +### ERRORS +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Ok(TokenStream [Ident { ident: "r", span: #25 bytes(460..484) }, Literal { kind: Char, symbol: "r", suffix: None, span: #25 bytes(460..484) }]) +Ok(TokenStream [Ident { ident: "c", span: #25 bytes(460..484) }, Literal { kind: Char, symbol: "r", suffix: None, span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: ErrWithGuar, symbol: "0b2", suffix: None, span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: ErrWithGuar, symbol: "0b", suffix: Some("f32"), span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: ErrWithGuar, symbol: "0b0.0", suffix: Some("f32"), span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: ErrWithGuar, symbol: "'''", suffix: None, span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: ErrWithGuar, symbol: "'\n'", suffix: None, span: #25 bytes(460..484) }]) +Ok(TokenStream [Literal { kind: ErrWithGuar, symbol: "0b2", suffix: None, span: #25 bytes(460..484) }]) +Ok(Literal { kind: ErrWithGuar, symbol: "0b2", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: ErrWithGuar, symbol: "0b", suffix: Some("f32"), span: #25 bytes(460..484) }) +Ok(Literal { kind: ErrWithGuar, symbol: "0b0.0", suffix: Some("f32"), span: #25 bytes(460..484) }) +Ok(Literal { kind: ErrWithGuar, symbol: "'''", suffix: None, span: #25 bytes(460..484) }) +Ok(Literal { kind: ErrWithGuar, symbol: "'\n'", suffix: None, span: #25 bytes(460..484) }) +Err(LexError) diff --git a/tests/ui/proc-macro/nonfatal-parsing.rs b/tests/ui/proc-macro/nonfatal-parsing.rs new file mode 100644 index 0000000000000..59d49692a3fd6 --- /dev/null +++ b/tests/ui/proc-macro/nonfatal-parsing.rs @@ -0,0 +1,26 @@ +//@ proc-macro: nonfatal-parsing.rs +//@ needs-unwind +//@ edition:2024 +//@ dont-require-annotations: ERROR +//@ revisions: in_macro standalone +//@[in_macro] check-fail +//@[standalone] run-pass +//@[standalone] check-run-results +// FIXME: in_macro should be a check-pass test +#![feature(proc_macro_standalone)] + +extern crate proc_macro; +extern crate nonfatal_parsing; + +#[path = "auxiliary/nonfatal-parsing-body.rs"] +mod body; + +fn main() { + #[cfg(in_macro)] + nonfatal_parsing::run!(); + + #[cfg(standalone)] + proc_macro::enable_standalone(); + #[cfg(standalone)] + body::run(); +} diff --git a/tests/ui/proc-macro/nonfatal-parsing.standalone.run.stderr b/tests/ui/proc-macro/nonfatal-parsing.standalone.run.stderr new file mode 100644 index 0000000000000..5a65484383869 --- /dev/null +++ b/tests/ui/proc-macro/nonfatal-parsing.standalone.run.stderr @@ -0,0 +1,8 @@ +1 ) 2 did not panic +( x [ ) ] did not panic +r# did not panic +r################################################################################################################################################################################################################################################################"a"################################################################################################################################################################################################################################################################ did not panic +1 ) 2 did not panic +( x [ ) ] did not panic +r# did not panic +r################################################################################################################################################################################################################################################################"a"################################################################################################################################################################################################################################################################ did not panic diff --git a/tests/ui/proc-macro/nonfatal-parsing.standalone.run.stdout b/tests/ui/proc-macro/nonfatal-parsing.standalone.run.stdout new file mode 100644 index 0000000000000..49c8846e3a31d --- /dev/null +++ b/tests/ui/proc-macro/nonfatal-parsing.standalone.run.stdout @@ -0,0 +1,62 @@ +Ok(Literal { kind: Float, symbol: "123", suffix: None, span: Span }) +Ok(Literal { kind: Str, symbol: "ab", suffix: None, span: Span }) +Err(LexError) +Err(LexError) +Ok(Literal { kind: ByteStr, symbol: "b", suffix: None, span: Span }) +Ok(Literal { kind: CStr, symbol: "b", suffix: None, span: Span }) +Ok(Literal { kind: CStrRaw(0), symbol: "b", suffix: None, span: Span }) +Err(LexError) +Ok(Literal { kind: Integer, symbol: "256", suffix: Some("u8"), span: Span }) +Ok(Literal { kind: Integer, symbol: "256", suffix: Some("u8"), span: Span }) +Ok(TokenStream []) +Ok(Literal { kind: Integer, symbol: "11111000000001111", suffix: Some("i16"), span: Span }) +Err(LexError) +Err(LexError) +Ok(Literal { kind: Float, symbol: "2E4", suffix: None, span: Span }) +Ok(Literal { kind: Float, symbol: "2.2E-4", suffix: Some("f64"), span: Span }) +Ok(Literal { kind: Float, symbol: "18u8E", suffix: None, span: Span }) +Ok(Literal { kind: Float, symbol: "18.0u8E", suffix: None, span: Span }) +Ok(Literal { kind: CStrRaw(1), symbol: "// /* // \n */", suffix: None, span: Span }) +Err(LexError) +Err(LexError) +Err(LexError) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +### ERRORS +Err(LexError) +Err(LexError) +Ok(Literal { kind: Float, symbol: "0 ", suffix: None, span: Span }) +Ok(Literal { kind: Float, symbol: "0//", suffix: None, span: Span }) +Ok(Literal { kind: Float, symbol: "3//\n4", suffix: None, span: Span }) +Ok(Literal { kind: Float, symbol: "18.u8E", suffix: None, span: Span }) +Err(LexError) +Err(LexError) +Err(LexError) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(TokenStream []) +Ok(Literal { kind: Float, symbol: "1 ) 2", suffix: None, span: Span }) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError) +Err(LexError)