Store file path and file id in token stream metadata (#1192)

commit-id:98fd5a13

---

**Stack**:
- #1200
- #1195
- #1194
- #1192 ⬅


⚠️ *Part of a stack created by [spr](https://github.com/ejoffe/spr). Do
not merge manually using the UI - doing so may have unexpected results.*

plugins/cairo-lang-macro-stable/src/lib.rs

@@ -1,24 +1,46 @@
 use crate::ffi::StableSlice;
-use std::ffi::{CStr, CString};
+use std::ffi::CStr;
 use std::num::NonZeroU8;
 use std::os::raw::c_char;
+use std::ptr::NonNull;
 
 pub mod ffi;
 
+/// An option.
+///
+/// This struct implements FFI-safe stable ABI.
+#[repr(C)]
+#[derive(Debug)]
+pub enum StableOption<T> {
+    None,
+    Some(T),
+}
+
 /// Token stream.
 ///
 /// This struct implements FFI-safe stable ABI.
 #[repr(C)]
 #[derive(Debug)]
-pub struct StableTokenStream(*mut c_char);
+pub struct StableTokenStream {
+    pub value: *mut c_char,
+    pub metadata: StableTokenStreamMetadata,
+}
 
+/// Token stream metadata.
+///
+/// This struct implements FFI-safe stable ABI.
 #[repr(C)]
 #[derive(Debug)]
-pub enum StableAuxData {
-    None,
-    Some(StableSlice<u8>),
+pub struct StableTokenStreamMetadata {
+    pub original_file_path: Option<NonNull<c_char>>,
+    pub file_id: Option<NonNull<c_char>>,
 }
 
+/// Auxiliary data returned by the procedural macro.
+///
+/// This struct implements FFI-safe stable ABI.
+pub type StableAuxData = StableOption<StableSlice<u8>>;
+
 /// Diagnostic returned by the procedural macro.
 ///
 /// This struct implements FFI-safe stable ABI.
@@ -63,29 +85,12 @@ pub struct StableResultWrapper {
 }
 
 impl StableTokenStream {
-    pub fn new(s: *mut c_char) -> Self {
-        Self(s)
-    }
-
     /// Convert to String.
     ///
     /// # Safety
     pub unsafe fn to_string(&self) -> String {
         // Note that this does not deallocate the c-string.
         // The memory must still be freed with `CString::from_raw`.
-        CStr::from_ptr(self.0).to_string_lossy().to_string()
-    }
-
-    pub fn into_owned_string(self) -> String {
-        unsafe { raw_to_string(self.0) }
-    }
-}
-
-unsafe fn raw_to_string(raw: *mut c_char) -> String {
-    if raw.is_null() {
-        String::default()
-    } else {
-        let cstr = CString::from_raw(raw);
-        cstr.to_string_lossy().to_string()
+        CStr::from_ptr(self.value).to_string_lossy().to_string()
     }
 }

plugins/cairo-lang-macro/Cargo.toml

@@ -16,3 +16,4 @@ repository.workspace = true
 cairo-lang-macro-attributes = { path = "../cairo-lang-macro-attributes" }
 cairo-lang-macro-stable = { path = "../cairo-lang-macro-stable" }
 linkme.workspace = true
+scarb-stable-hash = { path = "../../utils/scarb-stable-hash" }

plugins/cairo-lang-macro/src/lib.rs

@@ -5,10 +5,12 @@ pub use linkme;
 use cairo_lang_macro_stable::ffi::StableSlice;
 use cairo_lang_macro_stable::{
     StableAuxData, StableDiagnostic, StableProcMacroResult, StableSeverity, StableTokenStream,
+    StableTokenStreamMetadata,
 };
 use std::ffi::{c_char, CStr, CString};
 use std::fmt::Display;
 use std::num::NonZeroU8;
+use std::ptr::NonNull;
 use std::slice;
 use std::vec::IntoIter;
 
@@ -66,18 +68,49 @@ pub enum ProcMacroResult {
 }
 
 #[derive(Debug, Default, Clone)]
-pub struct TokenStream(String);
+pub struct TokenStream {
+    value: String,
+    metadata: TokenStreamMetadata,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct TokenStreamMetadata {
+    original_file_path: Option<String>,
+    file_id: Option<String>,
+}
 
 impl TokenStream {
     #[doc(hidden)]
-    pub fn new(s: String) -> Self {
-        Self(s)
+    pub fn new(value: String) -> Self {
+        Self {
+            value,
+            metadata: TokenStreamMetadata::default(),
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn with_metadata(mut self, metadata: TokenStreamMetadata) -> Self {
+        self.metadata = metadata;
+        self
+    }
+
+    pub fn metadata(&self) -> &TokenStreamMetadata {
+        &self.metadata
     }
 }
 
 impl Display for TokenStream {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.0)
+        write!(f, "{}", self.value)
+    }
+}
+
+impl TokenStreamMetadata {
+    pub fn new(file_path: impl ToString, file_id: impl ToString) -> Self {
+        Self {
+            original_file_path: Some(file_path.to_string()),
+            file_id: Some(file_id.to_string()),
+        }
     }
 }
 
@@ -332,8 +365,11 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub fn into_stable(self) -> StableTokenStream {
-        let cstr = CString::new(self.0).unwrap();
-        StableTokenStream::new(cstr.into_raw())
+        let cstr = CString::new(self.value).unwrap();
+        StableTokenStream {
+            value: cstr.into_raw(),
+            metadata: self.metadata.into_stable(),
+        }
     }
 
     /// Convert to native Rust representation, without taking the ownership of the string.
@@ -343,7 +379,10 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub unsafe fn from_stable(token_stream: &StableTokenStream) -> Self {
-        Self::new(token_stream.to_string())
+        Self {
+            value: from_raw_cstr(token_stream.value),
+            metadata: TokenStreamMetadata::from_stable(&token_stream.metadata),
+        }
     }
 
     /// Convert to native Rust representation, with taking the ownership of the string.
@@ -354,7 +393,64 @@ impl TokenStream {
     /// # Safety
     #[doc(hidden)]
     pub unsafe fn from_owned_stable(token_stream: StableTokenStream) -> Self {
-        Self::new(token_stream.into_owned_string())
+        Self {
+            value: from_raw_cstring(token_stream.value),
+            metadata: TokenStreamMetadata::from_owned_stable(token_stream.metadata),
+        }
+    }
+}
+
+impl TokenStreamMetadata {
+    /// Convert to FFI-safe representation.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub fn into_stable(self) -> StableTokenStreamMetadata {
+        let original_file_path = self
+            .original_file_path
+            .and_then(|path| NonNull::new(CString::new(path).unwrap().into_raw()));
+        let file_id = self
+            .file_id
+            .and_then(|path| NonNull::new(CString::new(path).unwrap().into_raw()));
+        StableTokenStreamMetadata {
+            original_file_path,
+            file_id,
+        }
+    }
+
+    /// Convert to native Rust representation, without taking the ownership of the string.
+    ///
+    /// Note that you still need to free the memory by calling `from_owned_stable`.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_stable(metadata: &StableTokenStreamMetadata) -> Self {
+        let original_file_path = metadata
+            .original_file_path
+            .map(|raw| from_raw_cstr(raw.as_ptr()));
+        let file_id = metadata.file_id.map(|raw| from_raw_cstr(raw.as_ptr()));
+        Self {
+            original_file_path,
+            file_id,
+        }
+    }
+
+    /// Convert to native Rust representation, with taking the ownership of the string.
+    ///
+    /// Useful when you need to free the allocated memory.
+    /// Only use on the same side of FFI-barrier, where the memory has been allocated.
+    ///
+    /// # Safety
+    #[doc(hidden)]
+    pub unsafe fn from_owned_stable(metadata: StableTokenStreamMetadata) -> Self {
+        let original_file_path = metadata
+            .original_file_path
+            .map(|raw| from_raw_cstring(raw.as_ptr()));
+        let file_id = metadata.file_id.map(|raw| from_raw_cstring(raw.as_ptr()));
+        Self {
+            original_file_path,
+            file_id,
+        }
     }
 }
 
@@ -495,3 +591,15 @@ unsafe fn from_raw_cstr(raw: *mut c_char) -> String {
         cstr.to_string_lossy().to_string()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::TokenStream;
+
+    #[test]
+    fn new_token_stream_metadata_empty() {
+        let token_stream = TokenStream::new("".to_string());
+        assert!(token_stream.metadata.file_id.is_none());
+        assert!(token_stream.metadata.original_file_path.is_none());
+    }
+}

scarb/src/compiler/plugin/proc_macro/host.rs

@@ -7,13 +7,16 @@ use cairo_lang_defs::plugin::{
     DynGeneratedFileAuxData, GeneratedFileAuxData, MacroPlugin, MacroPluginMetadata,
     PluginGeneratedFile, PluginResult,
 };
-use cairo_lang_macro::{AuxData, Diagnostic, ProcMacroResult, Severity, TokenStream};
+use cairo_lang_macro::{
+    AuxData, Diagnostic, ProcMacroResult, Severity, TokenStream, TokenStreamMetadata,
+};
 use cairo_lang_semantic::plugin::PluginSuite;
 use cairo_lang_syntax::attribute::structured::AttributeListStructurize;
 use cairo_lang_syntax::node::db::SyntaxGroup;
 use cairo_lang_syntax::node::ids::SyntaxStablePtrId;
 use cairo_lang_syntax::node::{ast, TypedSyntaxNode};
 use itertools::Itertools;
+use scarb_stable_hash::short_hash;
 use smol_str::SmolStr;
 use std::any::Any;
 use std::sync::Arc;
@@ -203,8 +206,11 @@ impl MacroPlugin for ProcMacroHostPlugin {
             .chain(self.handle_attribute(db, item_ast.clone()))
             .chain(self.handle_derive(db, item_ast.clone()));
         let stable_ptr = item_ast.clone().stable_ptr().untyped();
+        let file_path = stable_ptr.file_id(db).full_path(db.upcast());
+        let file_id = short_hash(file_path.clone());
 
-        let mut token_stream = TokenStream::from_item_ast(db, item_ast);
+        let mut token_stream = TokenStream::from_item_ast(db, item_ast)
+            .with_metadata(TokenStreamMetadata::new(file_path, file_id));
         let mut aux_data: Option<ProcMacroAuxData> = None;
         let mut modified = false;
         let mut all_diagnostics: Vec<Diagnostic> = Vec::new();

scarb/tests/build_cairo_plugin.rs

@@ -518,7 +518,7 @@ fn can_return_aux_data_from_plugin() {
         .build(&project);
 
     Scarb::quick_snapbox()
-        .arg("cairo-run")
+        .arg("build")
         // Disable output from Cargo.
         .env("CARGO_TERM_QUIET", "true")
         .current_dir(&project)
@@ -529,7 +529,57 @@ fn can_return_aux_data_from_plugin() {
             [..]Compiling hello v1.0.0 ([..]Scarb.toml)
             [SomeMacroDataFormat { msg: "Hello from some macro!" }]
             [..]Finished release target(s) in [..]
-            [..]Running hello
-            [..]Run completed successfully, returning [..]
+        "#});
+}
+
+#[test]
+fn can_read_token_stream_metadata() {
+    let temp = TempDir::new().unwrap();
+    let t = temp.child("some");
+    simple_project_with_code(
+        &t,
+        indoc! {r##"
+        use cairo_lang_macro::{ProcMacroResult, TokenStream, attribute_macro};
+
+        #[attribute_macro]
+        pub fn some_macro(token_stream: TokenStream) -> ProcMacroResult {
+            println!("{:#?}", token_stream.metadata());
+            ProcMacroResult::Leave { diagnostics: Vec::new() }
+        }
+
+        "##},
+    );
+
+    let project = temp.child("hello");
+    ProjectBuilder::start()
+        .name("hello")
+        .version("1.0.0")
+        .dep_starknet()
+        .dep("some", &t)
+        .lib_cairo(indoc! {r#"
+            #[some]
+            fn main() -> felt252 { 12 }
+        "#})
+        .build(&project);
+
+    Scarb::quick_snapbox()
+        .arg("build")
+        // Disable output from Cargo.
+        .env("CARGO_TERM_QUIET", "true")
+        .current_dir(&project)
+        .assert()
+        .success()
+        .stdout_matches(indoc! {r#"
+            [..]Compiling some v1.0.0 ([..]Scarb.toml)
+            [..]Compiling hello v1.0.0 ([..]Scarb.toml)
+            TokenStreamMetadata {
+                original_file_path: Some(
+                    "[..]lib.cairo",
+                ),
+                file_id: Some(
+                    "[..]",
+                ),
+            }
+            [..]Finished release target(s) in [..]
         "#});
 }