Dev

Cargo.toml

@@ -35,6 +35,7 @@ anyhow = "1"
 assert_fs = "1"
 async-trait = "0.1"
 axum = { version = "0.6", features = ["http2"] }
+bumpalo = "3"
 cairo-lang-casm = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" }
 cairo-lang-compiler = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" }
 cairo-lang-debug = { git = "https://github.com/starkware-libs/cairo", rev = "3874879adcc2f36d7e1235fc9aec8420ff7013b0" }

plugins/cairo-lang-macro-stable/src/lib.rs

@@ -1,11 +1,31 @@
 use crate::ffi::{StableOption, StableSlice};
-use std::ffi::CStr;
 use std::num::NonZeroU8;
 use std::os::raw::c_char;
 use std::ptr::NonNull;
 
 pub mod ffi;
 
+#[repr(C)]
+#[derive(Debug)]
+pub struct StableToken {
+    pub span: StableTextSpan,
+    pub ptr: *const u8,
+    pub len: usize,
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct StableTextSpan {
+    pub start: usize,
+    pub end: usize,
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub enum StableTokenTree {
+    Ident(StableToken),
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct StableExpansion {
@@ -23,8 +43,9 @@ pub type StableExpansionsList = StableSlice<StableExpansion>;
 #[repr(C)]
 #[derive(Debug)]
 pub struct StableTokenStream {
-    pub value: *mut c_char,
+    pub tokens: StableSlice<StableTokenTree>,
     pub metadata: StableTokenStreamMetadata,
+    pub size_hint: usize,
 }
 
 /// Token stream metadata.
@@ -35,6 +56,7 @@ pub struct StableTokenStream {
 pub struct StableTokenStreamMetadata {
     pub original_file_path: Option<NonNull<c_char>>,
     pub file_id: Option<NonNull<c_char>>,
+    pub edition: Option<NonNull<c_char>>,
 }
 
 /// Auxiliary data returned by the procedural macro.
@@ -76,17 +98,6 @@ pub struct StableResultWrapper {
     pub output: StableProcMacroResult,
 }
 
-impl StableTokenStream {
-    /// Convert to String.
-    ///
-    /// # Safety
-    pub unsafe fn to_string(&self) -> String {
-        // Note that this does not deallocate the c-string.
-        // The memory must still be freed with `CString::from_raw`.
-        CStr::from_ptr(self.value).to_string_lossy().to_string()
-    }
-}
-
 #[repr(C)]
 pub struct StablePostProcessContext {
     pub aux_data: StableSlice<StableAuxData>,

plugins/cairo-lang-macro/Cargo.toml

@@ -13,8 +13,9 @@ readme = "README.md"
 repository.workspace = true
 
 [dependencies]
-cairo-lang-macro-attributes = "0.1"
-cairo-lang-macro-stable = "1"
+bumpalo.workspace = true
+cairo-lang-macro-attributes = { path = "../cairo-lang-macro-attributes" }
+cairo-lang-macro-stable = { path = "../cairo-lang-macro-stable" }
 linkme.workspace = true
 serde = { workspace = true, optional = true }
 

plugins/cairo-lang-macro/src/lib.rs

@@ -18,17 +18,22 @@
 pub use cairo_lang_macro_attributes::*;
 #[doc(hidden)]
 pub use linkme;
+use std::cell::RefCell;
 
 use cairo_lang_macro_stable::ffi::StableSlice;
 use cairo_lang_macro_stable::{
     StableExpansionsList, StablePostProcessContext, StableProcMacroResult,
 };
 use std::ffi::{c_char, CStr, CString};
+use std::ops::Deref;
 
 mod types;
 
 pub use types::*;
 
+// A thread-local allocation context for allocating tokens on proc macro side.
+thread_local!(static CONTEXT: RefCell<AllocationContext> =  RefCell::default() );
+
 #[doc(hidden)]
 #[derive(Clone)]
 pub struct ExpansionDefinition {
@@ -97,29 +102,42 @@ pub unsafe extern "C" fn expand(
     stable_attr: cairo_lang_macro_stable::StableTokenStream,
     stable_token_stream: cairo_lang_macro_stable::StableTokenStream,
 ) -> cairo_lang_macro_stable::StableResultWrapper {
-    let token_stream = TokenStream::from_stable(&stable_token_stream);
-    let attr_token_stream = TokenStream::from_stable(&stable_attr);
-    let item_name = CStr::from_ptr(item_name).to_string_lossy().to_string();
-    let fun = MACRO_DEFINITIONS_SLICE
-        .iter()
-        .find_map(|m| {
-            if m.name == item_name.as_str() {
-                Some(m.fun.clone())
-            } else {
-                None
-            }
-        })
-        .expect("procedural macro not found");
-    let result = match fun {
-        ExpansionFunc::Attr(fun) => fun(attr_token_stream, token_stream),
-        ExpansionFunc::Other(fun) => fun(token_stream),
-    };
-    let result: StableProcMacroResult = result.into_stable();
-    cairo_lang_macro_stable::StableResultWrapper {
-        input: stable_token_stream,
-        input_attr: stable_attr,
-        output: result,
-    }
+    CONTEXT.with(|ctx_cell| {
+        // Read size hint from stable token stream. This will be used to create a sufficiently
+        // large bump allocation buffer.
+        let size_hint: usize = stable_token_stream.size_hint + stable_attr.size_hint;
+        // Replace the allocation context with a new one.
+        // If there is no interned string guards, the old context will be de-allocated.
+        ctx_cell.replace(AllocationContext::with_capacity(size_hint));
+        let ctx_borrow = ctx_cell.borrow();
+        let ctx: &AllocationContext = ctx_borrow.deref();
+        // Copy the stable token stream into current context.
+        let token_stream = TokenStream::from_stable_in(&stable_token_stream, ctx);
+        let attr_token_stream = TokenStream::from_stable_in(&stable_attr, ctx);
+        let item_name = CStr::from_ptr(item_name)
+            .to_str()
+            .expect("item name must be a valid string");
+        let fun = MACRO_DEFINITIONS_SLICE
+            .iter()
+            .find_map(|m| {
+                if m.name == item_name {
+                    Some(m.fun.clone())
+                } else {
+                    None
+                }
+            })
+            .expect("procedural macro not found");
+        let result = match fun {
+            ExpansionFunc::Attr(fun) => fun(attr_token_stream, token_stream),
+            ExpansionFunc::Other(fun) => fun(token_stream),
+        };
+        let result: StableProcMacroResult = result.into_stable();
+        cairo_lang_macro_stable::StableResultWrapper {
+            input: stable_token_stream,
+            input_attr: stable_attr,
+            output: result,
+        }
+    })
 }
 
 /// Free the memory allocated for the [`StableProcMacroResult`].
@@ -134,7 +152,7 @@ pub unsafe extern "C" fn expand(
 #[doc(hidden)]
 #[no_mangle]
 pub unsafe extern "C" fn free_result(result: StableProcMacroResult) {
-    ProcMacroResult::from_owned_stable(result);
+    ProcMacroResult::free_owned_stable(result);
 }
 
 /// Distributed slice for storing auxiliary data collection callback pointers.