feat: Collect referenced C# file sources (#516)

Il2cpp leaves markers in its generated C++ source to the C# files certain snippets
of code come from. We follow those snippets to also bundle up the C# files they
reference, as we would want to also allow resolving the C# source later on.

symbolic-debuginfo/Cargo.toml

@@ -104,6 +104,7 @@ zip = { version = "0.5.2", optional = true, default-features = false, features =
 [dev-dependencies]
 criterion = { version = "0.3.4", features = ["html_reports"] }
 insta = "1.3.0"
+tempfile = "3.1.0"
 similar-asserts = "1.0.0"
 symbolic-testutils = { path = "../symbolic-testutils" }
 

symbolic-debuginfo/src/sourcebundle.rs

@@ -757,6 +757,7 @@ where
 {
     manifest: SourceBundleManifest,
     writer: ZipWriter<W>,
+    collect_il2cpp: bool,
 }
 
 impl<W> SourceBundleWriter<W>
@@ -773,6 +774,7 @@ where
         Ok(SourceBundleWriter {
             manifest: SourceBundleManifest::new(),
             writer: ZipWriter::new(writer),
+            collect_il2cpp: false,
         })
     }
 
@@ -781,6 +783,12 @@ where
         self.manifest.files.is_empty()
     }
 
+    /// This controls if source files should be scanned for Il2cpp-specific source annotations,
+    /// and the referenced C# files should be bundled ups as well.
+    pub fn collect_il2cpp_sources(&mut self, collect_il2cpp: bool) {
+        self.collect_il2cpp = collect_il2cpp;
+    }
+
     /// Sets a meta data attribute of the bundle.
     ///
     /// Attributes are flushed to the bundle when it is [finished]. Thus, they can be retrieved or
@@ -911,6 +919,8 @@ where
         F: FnMut(&FileEntry) -> bool,
     {
         let mut files_handled = BTreeSet::new();
+        let mut referenced_files = BTreeSet::new();
+
         let session = object
             .debug_session()
             .map_err(|e| SourceBundleError::new(SourceBundleErrorKind::BadDebugFile, e))?;
@@ -935,7 +945,7 @@ where
             {
                 None
             } else {
-                File::open(&filename).ok().map(BufReader::new)
+                std::fs::read(&filename).ok()
             };
 
             if let Some(source) = source {
@@ -944,12 +954,31 @@ where
                 info.set_ty(SourceFileType::Source);
                 info.set_path(filename.clone());
 
-                self.add_file(bundle_path, source, info)?;
+                if self.collect_il2cpp {
+                    collect_il2cpp_sources(&source, &mut referenced_files);
+                }
+
+                self.add_file(bundle_path, source.as_slice(), info)?;
             }
 
             files_handled.insert(filename);
         }
 
+        for filename in referenced_files {
+            if files_handled.contains(&filename) {
+                continue;
+            }
+
+            if let Some(source) = File::open(&filename).ok().map(BufReader::new) {
+                let bundle_path = sanitize_bundle_path(&filename);
+                let mut info = SourceFileInfo::new();
+                info.set_ty(SourceFileType::Source);
+                info.set_path(filename.clone());
+
+                self.add_file(bundle_path, source, info)?;
+            }
+        }
+
         let is_empty = self.is_empty();
         self.finish()?;
 
@@ -1005,6 +1034,25 @@ where
     }
 }
 
+/// Processes the `source`, looking for `il2cpp` specific reference comments.
+///
+/// The files referenced by those comments are added to the `referenced_files` Set.
+fn collect_il2cpp_sources(source: &[u8], referenced_files: &mut BTreeSet<String>) {
+    if let Ok(source) = std::str::from_utf8(source) {
+        for line in source.lines() {
+            let line = line.trim();
+
+            if let Some(source_ref) = line.strip_prefix("//<source_info:") {
+                if let Some((file, _line)) = source_ref.rsplit_once(':') {
+                    if !referenced_files.contains(file) {
+                        referenced_files.insert(file.to_string());
+                    }
+                }
+            }
+        }
+    }
+}
+
 impl SourceBundleWriter<BufWriter<File>> {
     /// Create a bundle writer that writes its output to the given path.
     ///
@@ -1033,6 +1081,7 @@ mod tests {
     use std::io::Cursor;
 
     use similar_asserts::assert_eq;
+    use tempfile::NamedTempFile;
 
     #[test]
     fn test_has_file() -> Result<(), SourceBundleError> {
@@ -1060,6 +1109,70 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_il2cpp_reference() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cpp_file = NamedTempFile::new()?;
+        let mut cs_file = NamedTempFile::new()?;
+
+        let cpp_contents = format!("foo\n//<source_info:{}:111>\nbar", cs_file.path().display());
+
+        // well, a source bundle itself is an `ObjectLike` :-)
+        let object_buf = {
+            let mut writer = Cursor::new(Vec::new());
+            let mut bundle = SourceBundleWriter::start(&mut writer)?;
+
+            let path = cpp_file.path().to_string_lossy();
+            let mut info = SourceFileInfo::new();
+            info.set_ty(SourceFileType::Source);
+            info.set_path(path.to_string());
+            bundle.add_file(path, cpp_contents.as_bytes(), info)?;
+
+            bundle.finish()?;
+            writer.into_inner()
+        };
+        let object = SourceBundle::parse(&object_buf)?;
+
+        // write file contents to temp files
+        cpp_file.write_all(cpp_contents.as_bytes())?;
+        cs_file.write_all(b"some C# source")?;
+
+        // write the actual source bundle based on the `object`
+        let mut output_buf = Cursor::new(Vec::new());
+        let mut writer = SourceBundleWriter::start(&mut output_buf)?;
+        writer.collect_il2cpp_sources(true);
+
+        let written = writer.write_object(&object, "whatever")?;
+        assert!(written);
+        let output_buf = output_buf.into_inner();
+
+        // and collect all the included files
+        let source_bundle = SourceBundle::parse(&output_buf)?;
+        let session = source_bundle.debug_session()?;
+        let actual_files: BTreeMap<_, _> = session
+            .files()
+            .flatten()
+            .flat_map(|f| {
+                let path = f.abs_path_str();
+                session
+                    .source_by_path(&path)
+                    .ok()
+                    .flatten()
+                    .map(|c| (path, c.into_owned()))
+            })
+            .collect();
+
+        let mut expected_files = BTreeMap::new();
+        expected_files.insert(cpp_file.path().to_string_lossy().into_owned(), cpp_contents);
+        expected_files.insert(
+            cs_file.path().to_string_lossy().into_owned(),
+            String::from("some C# source"),
+        );
+
+        assert_eq!(actual_files, expected_files);
+
+        Ok(())
+    }
+
     #[test]
     fn test_bundle_paths() {
         assert_eq!(sanitize_bundle_path("foo"), "foo");

symbolic-il2cpp/src/line_mapping/from_object.rs

@@ -1,15 +1,19 @@
+use std::collections::BTreeMap;
+use std::io::Write;
 use std::iter::Enumerate;
 use std::str::Lines;
-use std::{collections::BTreeMap, io::Write};
 
-use symbolic_common::ByteView;
+use symbolic_common::{ByteView, DebugId};
 use symbolic_debuginfo::{DebugSession, ObjectLike};
 
 /// A line mapping extracted from an object.
 ///
 /// This is only intended as an intermediate structure for serialization,
 /// not for lookups.
-pub struct ObjectLineMapping(BTreeMap<String, BTreeMap<String, BTreeMap<u32, u32>>>);
+pub struct ObjectLineMapping {
+    mapping: BTreeMap<String, BTreeMap<String, BTreeMap<u32, u32>>>,
+    debug_id: DebugId,
+}
 
 impl ObjectLineMapping {
     /// Create a line mapping from the given `object`.
@@ -21,6 +25,7 @@ impl ObjectLineMapping {
         O: ObjectLike<'data, 'object, Error = E>,
     {
         let session = object.debug_session()?;
+        let debug_id = object.debug_id();
 
         let mut mapping = BTreeMap::new();
 
@@ -51,16 +56,29 @@ impl ObjectLineMapping {
             }
         }
 
-        Ok(Self(mapping))
+        Ok(Self { mapping, debug_id })
     }
 
     /// Serializes the line mapping to the given writer as JSON.
     ///
     /// The mapping is serialized in the form of nested objects:
     /// C++ file => C# file => C++ line => C# line
-    pub fn to_writer<W: Write>(&self, writer: &mut W) -> std::io::Result<()> {
-        serde_json::to_writer(writer, &self.0)?;
-        Ok(())
+    ///
+    /// Returns `false` if the resulting JSON did not contain any mappings.
+    pub fn to_writer<W: Write>(mut self, writer: &mut W) -> std::io::Result<bool> {
+        let is_empty = self.mapping.is_empty();
+
+        // This is a big hack: We need the files for different architectures to be different.
+        // To achieve this, we put the debug-id of the file (which is different between architectures)
+        // into the same structure as the normal map, like so:
+        // `"__debug-id__": {"00000000-0000-0000-0000-000000000000": {}}`
+        // When parsing via `LineMapping::parse`, this *looks like* a valid entry, but we will
+        // most likely never have a C++ file named `__debug-id__` ;-)
+        let value = BTreeMap::from([(self.debug_id.to_string(), Default::default())]);
+        self.mapping.insert("__debug-id__".to_owned(), value);
+
+        serde_json::to_writer(writer, &self.mapping)?;
+        Ok(!is_empty)
     }
 }
 

symbolic-il2cpp/src/line_mapping/mod.rs

@@ -25,6 +25,12 @@ impl LineMapping {
 
         if let serde_json::Value::Object(object) = json {
             for (cpp_file, file_map) in object {
+                // This is a sentinel value for the originating debug file, which
+                // `ObjectLineMapping::to_writer` writes to the file to make it unique
+                // (and dependent on the originating debug-id).
+                if cpp_file == "__debug-id__" {
+                    continue;
+                }
                 let mut lines = Vec::new();
                 if let serde_json::Value::Object(file_map) = file_map {
                     for (cs_file, line_map) in file_map {