From 021a6710bd7b0bd9bfbd5ad67f6f809ad4226c41 Mon Sep 17 00:00:00 2001 From: George Pollard Date: Tue, 1 Aug 2023 21:48:49 +0000 Subject: [PATCH] Remove srcview --- src/agent/Cargo.lock | 15 - src/agent/Cargo.toml | 1 - src/agent/srcview/.gitignore | 1 - src/agent/srcview/Cargo.toml | 21 - src/agent/srcview/README.md | 34 -- src/agent/srcview/TODO.md | 6 - src/agent/srcview/examples/dump_cobertura.rs | 59 -- src/agent/srcview/examples/dump_modoff.rs | 19 - src/agent/srcview/examples/dump_paths.rs | 27 - src/agent/srcview/examples/dump_srcloc.rs | 31 - src/agent/srcview/res/example.txt | 15 - src/agent/srcview/src/bin/srcview.rs | 197 ------- src/agent/srcview/src/lib.rs | 105 ---- src/agent/srcview/src/modoff.rs | 229 -------- src/agent/srcview/src/pdbcache.rs | 120 ---- src/agent/srcview/src/report.rs | 570 ------------------- src/agent/srcview/src/srcline.rs | 46 -- src/agent/srcview/src/srcview.rs | 286 ---------- src/agent/srcview/tests/modoff.rs | 37 -- src/agent/srcview/tests/srcview.rs | 77 --- src/ci/agent.sh | 3 - src/ci/check-dependencies.sh | 25 - 22 files changed, 1924 deletions(-) delete mode 100644 src/agent/srcview/.gitignore delete mode 100644 src/agent/srcview/Cargo.toml delete mode 100644 src/agent/srcview/README.md delete mode 100644 src/agent/srcview/TODO.md delete mode 100644 src/agent/srcview/examples/dump_cobertura.rs delete mode 100644 src/agent/srcview/examples/dump_modoff.rs delete mode 100644 src/agent/srcview/examples/dump_paths.rs delete mode 100644 src/agent/srcview/examples/dump_srcloc.rs delete mode 100644 src/agent/srcview/res/example.txt delete mode 100644 src/agent/srcview/src/bin/srcview.rs delete mode 100644 src/agent/srcview/src/lib.rs delete mode 100644 src/agent/srcview/src/modoff.rs delete mode 100644 src/agent/srcview/src/pdbcache.rs delete mode 100644 src/agent/srcview/src/report.rs delete mode 100644 src/agent/srcview/src/srcline.rs delete mode 100644 src/agent/srcview/src/srcview.rs delete mode 100644 src/agent/srcview/tests/modoff.rs delete mode 100644 src/agent/srcview/tests/srcview.rs diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 40f8e42f8f..28a3ca63e5 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -3347,21 +3347,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "srcview" -version = "0.1.2" -dependencies = [ - "anyhow", - "clap", - "env_logger", - "log", - "nom", - "pdb", - "quick-xml 0.30.0", - "regex", - "serde", -] - [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index c3a7ba2276..bd09883ac7 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -13,7 +13,6 @@ members = [ "onefuzz-file-format", "onefuzz-telemetry", "reqwest-retry", - "srcview", "storage-queue", "win-util", "libclusterfuzz", diff --git a/src/agent/srcview/.gitignore b/src/agent/srcview/.gitignore deleted file mode 100644 index 8bda2718d5..0000000000 --- a/src/agent/srcview/.gitignore +++ /dev/null @@ -1 +0,0 @@ -res/example.pdb diff --git a/src/agent/srcview/Cargo.toml b/src/agent/srcview/Cargo.toml deleted file mode 100644 index 2eccef59f7..0000000000 --- a/src/agent/srcview/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "srcview" -version = "0.1.2" -edition = "2018" -license = "MIT" - -[features] -# Feature to gate tests that depend on binary artifacts, should be removed when -# #1143 is resolved. -binary-tests = [] - -[dependencies] -log = "0.4" -nom = "7" -pdb = "0.8" -regex = "1" -serde = { version = "1", features = ["derive"] } -quick-xml = "0.30" -anyhow = "1.0" -env_logger = "0.10" -clap = { version = "4.3.0", features = ["derive"] } diff --git a/src/agent/srcview/README.md b/src/agent/srcview/README.md deleted file mode 100644 index f34045fad2..0000000000 --- a/src/agent/srcview/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# srcview - -A library for mapping module+offset to source:line. Note that you'll get -significantly better results if you use instruction level coverage as -opposed to branch. Alternatively you're welcome to post-process branch -coverage into instruction, but this project does not assist with that. - -## Docs - -`cargo doc --open` - -## Usage - -See [`examples/dump_cobertura.rs`](examples/dump_cobertura.rs). - -This can be run with `cargo run --example dump_cobertura res\example.pdb res\example.txt`. - -The biggest challenge when making this work is likely getting the absolute PDB -paths to match relative paths inside your repo. To make this a bit easier, you -can dump the PDB paths with: `cargo run --example dump_paths res\example.pdb` - -## ADO Integration - -See [`azure-pipelines.yml`](azure-pipelines.yml). - -## VSCode Integration - -Install [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters), -then place a file named `coverage.xml` at a location such that the relative -file paths match your repo. Note that this is a file you should generate -yourself, and that the `coverage.xml` included in this repo will probably not -help you. From there, navigate to any file you expect to see coverage in. If -the paths and coverage file correctly line up, you should see red or green bars -next to the source lines. \ No newline at end of file diff --git a/src/agent/srcview/TODO.md b/src/agent/srcview/TODO.md deleted file mode 100644 index 100e8096fb..0000000000 --- a/src/agent/srcview/TODO.md +++ /dev/null @@ -1,6 +0,0 @@ -# TODO -- add trace/info/warn -- flesh out modoff parser error -- consider using Cow to reduce memory usage -- consider mixed case module names? -- redo xml generation to not be hand-rolled \ No newline at end of file diff --git a/src/agent/srcview/examples/dump_cobertura.rs b/src/agent/srcview/examples/dump_cobertura.rs deleted file mode 100644 index fa5f1e77e9..0000000000 --- a/src/agent/srcview/examples/dump_cobertura.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::io::{stdout, BufWriter}; -use std::path::Path; -use std::{env, fs, process}; - -use srcview::{ModOff, Report, SrcLine, SrcView}; - -fn main() { - let args = env::args().collect::>(); - - if args.len() != 3 { - eprintln!("Usage: {} ", args[0]); - process::exit(1); - } - - let pdb_path = Path::new(&args[1]); - let modoff_path = Path::new(&args[2]); - - // read our modoff file and parse it to a vector - let modoff_data = fs::read_to_string(modoff_path).unwrap(); - let modoffs = ModOff::parse(&modoff_data).unwrap(); - - // create all the likely module base names -- do we care about mixed case - // here? - let bare = pdb_path.file_stem().unwrap().to_string_lossy(); - let exe = format!("{bare}.exe"); - let dll = format!("{bare}.dll"); - let sys = format!("{bare}.sys"); - - // create our new SrcView and insert our only pdb into it - // we don't know what the modoff module will be, so create a mapping from - // all likely names to the pdb - - let mut srcview = SrcView::new(); - - // in theory we could refcount the pdbcache's to save resources here, but - // Im not sure thats necesary... - srcview.insert(&bare, pdb_path).unwrap(); - srcview.insert(&exe, pdb_path).unwrap(); - srcview.insert(&dll, pdb_path).unwrap(); - srcview.insert(&sys, pdb_path).unwrap(); - - // Convert our ModOffs to SrcLine so we can draw it - let coverage: Vec = modoffs - .into_iter() - .filter_map(|m| srcview.modoff(&m)) - .collect(); - - // Generate our report, filtering on our example path - let r = Report::new(&coverage, &srcview, Some(r"E:\\1f\\coverage\\example")).unwrap(); - - // Format it as cobertura and display it - // output is built incrementally so buffer it - let mut buffered_stdout = BufWriter::new(stdout().lock()); - r.cobertura(Some(r"E:\\1f\\coverage\\"), &mut buffered_stdout) - .unwrap(); -} diff --git a/src/agent/srcview/examples/dump_modoff.rs b/src/agent/srcview/examples/dump_modoff.rs deleted file mode 100644 index f8fb9a63a0..0000000000 --- a/src/agent/srcview/examples/dump_modoff.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::{env, fs, process}; - -use srcview::ModOff; - -fn main() { - let args = env::args().collect::>(); - - if args.len() != 2 { - eprintln!("Usage: {} ", args[0]); - process::exit(1); - } - - let modoff = fs::read_to_string(&args[1]).unwrap(); - - println!("{:#?}", ModOff::parse(&modoff).unwrap()); -} diff --git a/src/agent/srcview/examples/dump_paths.rs b/src/agent/srcview/examples/dump_paths.rs deleted file mode 100644 index 8758ddb7fd..0000000000 --- a/src/agent/srcview/examples/dump_paths.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::path::PathBuf; -use std::{env, process}; - -use srcview::SrcView; - -fn main() { - let args = env::args().collect::>(); - - if args.len() != 2 { - eprintln!("Usage: {} ", args[0]); - process::exit(1); - } - - let pdb_path = PathBuf::from(&args[1]); - - let mut srcview = SrcView::new(); - srcview - .insert(pdb_path.file_stem().unwrap().to_str().unwrap(), &pdb_path) - .unwrap(); - - for path in srcview.paths() { - println!("{}", path.display()); - } -} diff --git a/src/agent/srcview/examples/dump_srcloc.rs b/src/agent/srcview/examples/dump_srcloc.rs deleted file mode 100644 index d8475dbaf4..0000000000 --- a/src/agent/srcview/examples/dump_srcloc.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::{env, fs, process}; - -use srcview::{ModOff, SrcView}; - -fn main() { - let args = env::args().collect::>(); - - if args.len() != 3 { - eprintln!("Usage: {} ", args[0]); - process::exit(1); - } - - let pdb_path = &args[1]; - let modoff_path = &args[2]; - - let modoff_data = fs::read_to_string(modoff_path).unwrap(); - let modoffs = ModOff::parse(&modoff_data).unwrap(); - let mut srcview = SrcView::new(); - srcview.insert("example.exe", pdb_path).unwrap(); - - for modoff in &modoffs { - print!(" +{:04x} ", modoff.offset); - match srcview.modoff(modoff) { - Some(srcloc) => println!("{srcloc}"), - None => println!(), - } - } -} diff --git a/src/agent/srcview/res/example.txt b/src/agent/srcview/res/example.txt deleted file mode 100644 index 399ce4e29a..0000000000 --- a/src/agent/srcview/res/example.txt +++ /dev/null @@ -1,15 +0,0 @@ -example.exe+00006f70 -example.exe+00006f75 -example.exe+00006f79 -example.exe+00006f7d -example.exe+00006f81 -example.exe+00006f82 -example.exe+00006f85 -example.exe+00006f87 -example.exe+00006f89 -example.exe+00006f8b -example.exe+00006f9b -example.exe+00006fa2 -example.exe+00006fa7 -example.exe+00006fa9 -example.exe+00006fad diff --git a/src/agent/srcview/src/bin/srcview.rs b/src/agent/srcview/src/bin/srcview.rs deleted file mode 100644 index 7c9b84125a..0000000000 --- a/src/agent/srcview/src/bin/srcview.rs +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use anyhow::{format_err, Context, Result}; -use clap::Parser; -use srcview::{ModOff, Report, SrcLine, SrcView}; -use std::fs::{self, OpenOptions}; -use std::io::{stdout, BufWriter, Write}; -use std::path::{Path, PathBuf}; - -#[derive(Parser, Debug)] -enum Opt { - Srcloc(SrcLocOpt), - PdbPaths(PdbPathsOpt), - Cobertura(CoberturaOpt), - /// Print 3rd-party license information - Licenses, -} - -/// Print the file paths in the provided PDB -#[derive(Parser, Debug)] -struct PdbPathsOpt { - pdb_path: PathBuf, -} - -/// Print modoffset file with file and source lines -#[derive(Parser, Debug)] -struct SrcLocOpt { - pdb_path: PathBuf, - modoff_path: PathBuf, - #[arg(long)] - module_name: Option, -} - -/// Generate a Cobertura XML coverage report -/// -/// Example: -/// srcview cobertura ./res/example.pdb res/example.txt - -/// --include-regex "E:\\\\1f\\\\coverage\\\\" -/// --filter-regex "E:\\\\1f\\\\coverage\\\\" -/// --module-name example.exe -/// -/// In this example, only files that live in E:\1f\coverage are included and -/// E:\1f\coverage is removed from the filenames in the resulting XML report. -/// -/// The XML report is written to either a file or stdout if the argument is -/// a single dash. -#[derive(Parser, Debug)] -struct CoberturaOpt { - pdb_path: PathBuf, - modoff_path: PathBuf, - #[arg(default_value = "-")] - output_path: String, - #[arg(long)] - module_name: Option, - - /// regular expression that will be applied against the file paths from the - /// srcview - #[arg(long)] - include_regex: Option, - - /// search and replace regular expression that is applied to all file - /// paths that will appear in the output report - #[arg(long)] - filter_regex: Option, -} - -fn main() -> Result<()> { - env_logger::init(); - - let opt = Opt::parse(); - - match opt { - Opt::Srcloc(opts) => srcloc(opts)?, - Opt::PdbPaths(opts) => pdb_paths(opts)?, - Opt::Cobertura(opts) => cobertura(opts)?, - Opt::Licenses => licenses()?, - }; - - Ok(()) -} - -fn licenses() -> Result<()> { - stdout().write_all(include_bytes!("../../../data/licenses.json"))?; - Ok(()) -} - -// In the case the user did not specify the module name of interest, this -// utility function will guess at the module name based on the PDB path name. -// -// This is a last-ditch effort to ensure the coverage report has something -// consumable. -fn add_common_extensions(srcview: &mut SrcView, pdb_path: &Path) -> Result<()> { - let pdb_file_name = pdb_path.file_name().ok_or_else(|| { - format_err!( - "unable to identify file name from path: {}", - pdb_path.display() - ) - })?; - - let stem = Path::new(pdb_file_name) - .file_stem() - .ok_or_else(|| { - format_err!( - "unable to identify file stem from path: {}", - pdb_path.display() - ) - })? - .to_string_lossy(); - - // add module without extension - srcview.insert(&stem, pdb_path)?; - // add common module extensions - for ext in ["sys", "exe", "dll"] { - srcview.insert(&format!("{stem}.{ext}"), pdb_path)?; - } - Ok(()) -} - -fn srcloc(opts: SrcLocOpt) -> Result<()> { - let modoff_data = fs::read_to_string(&opts.modoff_path) - .with_context(|| format!("unable to read modoff_path: {}", opts.modoff_path.display()))?; - let modoffs = ModOff::parse(&modoff_data)?; - let mut srcview = SrcView::new(); - - if let Some(module_name) = &opts.module_name { - srcview.insert(module_name, &opts.pdb_path)?; - } else { - add_common_extensions(&mut srcview, &opts.pdb_path)?; - } - - for modoff in &modoffs { - print!(" +{:04x} ", modoff.offset); - match srcview.modoff(modoff) { - Some(srcloc) => println!("{srcloc}"), - None => println!(), - } - } - - Ok(()) -} - -fn pdb_paths(opts: PdbPathsOpt) -> Result<()> { - let mut srcview = SrcView::new(); - srcview.insert(&opts.pdb_path.to_string_lossy(), &opts.pdb_path)?; - - for path in srcview.paths() { - println!("{}", path.display()); - } - Ok(()) -} - -fn cobertura(opts: CoberturaOpt) -> Result<()> { - // read our modoff file and parse it to a vector - let modoff_data = fs::read_to_string(&opts.modoff_path)?; - let modoffs = ModOff::parse(&modoff_data)?; - - let mut output_writer = match opts.output_path.as_str() { - "-" => Box::new(BufWriter::new(stdout())) as Box, - path => { - let path = Path::new(path); - - Box::new(BufWriter::with_capacity( - 0x10_0000, // 1MB - OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(path)?, - )) as Box - } - }; - - // create our new SrcView and insert our only pdb into it - // we don't know what the modoff module will be, so create a mapping from - // all likely names to the pdb - let mut srcview = SrcView::new(); - - if let Some(module_name) = &opts.module_name { - srcview.insert(module_name, &opts.pdb_path)?; - } else { - add_common_extensions(&mut srcview, &opts.pdb_path)?; - } - - // Convert our ModOffs to SrcLine so we can draw it - let coverage: Vec = modoffs - .into_iter() - .filter_map(|m| srcview.modoff(&m)) - .collect(); - - // Generate our report, filtering on our example path - let r = Report::new(&coverage, &srcview, opts.include_regex.as_deref())?; - - // Format it as cobertura and display it - r.cobertura(opts.filter_regex.as_deref(), &mut output_writer)?; - Ok(()) -} diff --git a/src/agent/srcview/src/lib.rs b/src/agent/srcview/src/lib.rs deleted file mode 100644 index ff7f6bd9c3..0000000000 --- a/src/agent/srcview/src/lib.rs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -//! # srcview -//! -//! srcview is a crate for converting module+offset (modoff) coverage traces to source and line -//! number such that they can be visualized in an editor web frontend. It's job is split into -//! two parts: first creating a `SrcView` from debug info, then turning that `SrcView` and coverage into -//! a `Report`. -//! -//! ## SrcView -//! -//! There are two fundamental datatypes the crate defines, `ModOff` and `SrcLine`. `ModOff` is a module -//! name + offset from it's base, and SrcLine is an absolute path and line number. From a high -//! level, srcview converts modoff to srcline using debug info (PDBs only presently), then collects -//! and formats that debug info into a report. A `SrcView` is a collection of debug info (PdbCache) -//! mappings, and each debug info exposes four mappings: -//! - ModOff to SrcLine (1:1) -//! - Path to all valid SrcLines (1:n) -//! - Path to all Symbols in that file (1:n) -//! - Symbol to all valid SrcLines (1:n) -//! -//! A SrcView is a queryable collection of debug information, it _does not_ contain any coverage -//! information directly. It is simply the data extracted from the relevant debug structures in a -//! conviently queryable form. -//! -//! ## SrcView Examples -//! -//! ```text -//! module offset -//! -//! ┌─────────┐ -//! │ │ ┌──────────────┐ -//! │ SrcView │ ─────┬───►│PdbCache - foo│ -//! │ │ │ └──────────────┘ ┌──────────────────────────┐ -//! └─────────┘ │ ┌──►│SrcLine - z:\src\fizz.c:41│ -//! │ ┌──────────────┐ │ │SrcLine - z:\src\fizz.c:42│ -//! ├───►│PdbCache - bar├─────┤ └──────────────────────────┘ -//! │ └──────────────┘ │ -//! │ │ ┌──────────────────────────┐ -//! │ ┌──────────────┐ └──►│SrcLine - z:\src\quux.c:65│ -//! └───►│PdbCache - baz│ └──────────────────────────┘ -//! └──────────────┘ -//! ``` -//! -//! For example, bar+1234 might map to z:\src\quux.c:65. -//! -//! Absolute path to all valid SrcLine or Symbols in that path: -//! ```text -//! path -//! ┌─────────┐ -//! │ │ ┌──────────────┐ -//! │ SrcView │ ─────┬───►│PdbCache - foo│ -//! │ │ │ └──────────────┘ ┌──────────────────────────┐ -//! └─────────┘ │ ┌──►│SrcLine - z:\src\fizz.c:41│ -//! │ ┌──────────────┐ │ │SrcLine - z:\src\fizz.c:42│ -//! ├───►│PdbCache - bar├─────┤ └──────────────────────────┘ -//! │ └──────────────┘ │ -//! │ │ ┌──────────────────────────┐ -//! │ ┌──────────────┐ └──►│SrcLine - z:\src\quux.c:65│ -//! └───►│PdbCache - baz├──┐ └──────────────────────────┘ -//! └──────────────┘ │ -//! │ ┌──────────────────────────┐ -//! └─────►│SrcLine - z:\src\fizz.c:43│ -//! └──────────────────────────┘ -//! ``` -//! -//! For example if we were asking for SrcLines, z:\src\fizz.c would iterate over all -//! pdbcache's and return a list of z:\src\fizz.c:41, z:\src\fizz.c:42, z:\src.fizz.c:65. -//! If we were asking for a list of symbols for a path, we might get back -//! FunctionOne, FunctionTwo, SomeGlobal, etc. -//! -//! Finally, we can query for symbol (e.g. `foo!FunctionOne`) to `SrcLine`. This functions -//! exactly like the others and will return a collection of `SrcLine`. -//! -//! ## Report -//! -//! Once we have a `SrcView`, we can combine that with the coverage set (i.e. `&[ModOff]`) to create -//! a `Report`. A `Report` is fundamentally the same info as a SrcView, except keyed around file paths -//! and with some statistics computed from the coverage info. -//! -//! Additionally, `Report` handles some of the messier parts: -//! - Using regex's to filter `SrcView` contents, this lets you exclude things like the CRT from your -//! output reports. -//! - Using regex's to fixup input paths. All paths from PDBs are absolute, but most coverage -//! visualization wants them to match a directory path. For example, if you have a git repo in -//! ADO that is named 'test', it might be built on a build machine under 'z:\src\test`. To get -//! the paths to match, we need to filter off 'z:\src\test'. -//! - The actually emission of the coverage report itself. Right now this is only in Cobertura, but -//! its not hard to imagine other formats being added. -//! - Compute the coverage statistics on directories -//! -//! `Report` is significantly messier than `SrcView` and as of writing this I expect there to still be bugs. -//! -mod modoff; -mod pdbcache; -mod report; -mod srcline; -mod srcview; - -pub use self::srcview::SrcView; -pub use modoff::{ModOff, ModOffParseError}; -pub use pdbcache::PdbCache; -pub use report::Report; -pub use srcline::SrcLine; diff --git a/src/agent/srcview/src/modoff.rs b/src/agent/srcview/src/modoff.rs deleted file mode 100644 index c067588af6..0000000000 --- a/src/agent/srcview/src/modoff.rs +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::cmp::Ordering; -use std::error::Error; -use std::fmt; - -use log::*; - -use nom::bytes::complete::{tag, take_till1, take_while}; -use nom::character::complete::line_ending; -use nom::combinator::{eof, map_res, opt}; -use nom::multi::many0; -use nom::IResult; - -/// A module name and an offset -#[derive(Clone, Eq, PartialEq, Hash)] -pub struct ModOff { - pub module: String, - pub offset: usize, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub enum ModOffParseError { - InvalidFormat, -} - -impl From>> for ModOffParseError { - fn from(_: nom::Err>) -> Self { - ModOffParseError::InvalidFormat - } -} - -impl Error for ModOffParseError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - // TODO percolate up the nom error... - None - } -} - -impl fmt::Display for ModOffParseError { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "invalid modoff") - } -} - -impl fmt::Debug for ModOff { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.debug_struct("ModOff") - .field("module", &self.module) - .field("offset", &format_args!("{:#x}", self.offset)) - .finish() - } -} - -impl fmt::Display for ModOff { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "{}+{:x}", &self.module, self.offset) - } -} - -impl Ord for ModOff { - fn cmp(&self, other: &Self) -> Ordering { - let path_cmp = self.module.cmp(&other.module); - - if path_cmp != Ordering::Equal { - return path_cmp; - } - - self.offset.cmp(&other.offset) - } -} - -impl PartialOrd for ModOff { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl ModOff { - pub fn new(module: &str, offset: usize) -> Self { - Self { - module: module.to_owned(), - offset, - } - } - - fn parse_module(input: &str) -> IResult<&str, String> { - let (input, module) = take_till1(|c| c == '+')(input)?; - - Ok((input, module.to_owned())) - } - - fn from_hex(input: &str) -> Result { - usize::from_str_radix(input, 16) - } - - fn is_hex_digit(c: char) -> bool { - c.is_ascii_hexdigit() - } - - fn parse_offset(input: &str) -> IResult<&str, usize> { - let (input, _) = opt(tag("0x"))(input)?; - map_res(take_while(Self::is_hex_digit), Self::from_hex)(input) - } - - fn parse_modoff(input: &str) -> IResult<&str, Self> { - // TODO add modoff comment support -- lines starting with # or ; - let (input, module) = Self::parse_module(input)?; - let (input, _) = tag("+")(input)?; - let (input, offset) = Self::parse_offset(input)?; - let (input, _) = opt(line_ending)(input)?; - - Ok((input, Self { module, offset })) - } - - /// Parse a newline separate string of modoffs to a `Vec` - /// - /// # Arguments - /// - /// * `input` - A string containing new line separated '+' - /// - /// # Errors - /// - /// If the input string contains invalid module+offset - /// - /// # Example - /// ``` - /// use srcview::ModOff; - /// - /// assert_eq!( - /// vec![ - /// ModOff::new("foo.exe", 0x4141), - /// ModOff::new("foo.exe", 0x4242) - /// ], - /// ModOff::parse("foo.exe+4141\nfoo.exe+4242").unwrap() - /// ); - /// ``` - pub fn parse(input: &str) -> Result, ModOffParseError> { - let (input, res) = many0(Self::parse_modoff)(input)?; - let (_, _) = eof(input)?; - - info!("parsed {} modoff entries", res.len()); - - Ok(res) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use anyhow::Result; - - #[test] - fn parse_empty() -> Result<()> { - let empty: Vec = Vec::new(); - assert_eq!(empty, ModOff::parse("")?); - Ok(()) - } - - #[test] - fn parse_good() -> Result<()> { - assert_eq!( - vec![ModOff::new("foo.exe", 0x4141)], - ModOff::parse("foo.exe+4141")? - ); - Ok(()) - } - - #[test] - fn parse_good_multiple_unix() -> Result<()> { - assert_eq!( - vec![ - ModOff::new("foo.exe", 0x4141), - ModOff::new("foo.exe", 0x4242) - ], - ModOff::parse("foo.exe+4141\nfoo.exe+4242")? - ); - Ok(()) - } - - #[test] - fn parse_good_multiple_windows() -> Result<()> { - assert_eq!( - vec![ - ModOff::new("foo.exe", 0x4141), - ModOff::new("foo.exe", 0x4242), - ], - ModOff::parse("foo.exe+4141\r\nfoo.exe+4242")? - ); - Ok(()) - } - - #[test] - fn parse_good_leading_0x() -> Result<()> { - assert_eq!( - vec![ModOff::new("foo.exe", 0x4141)], - ModOff::parse("foo.exe+0x4141")? - ); - Ok(()) - } - - #[test] - fn parse_bad_no_module() { - assert_eq!(Err(ModOffParseError::InvalidFormat), ModOff::parse("+4141")); - } - #[test] - fn parse_bad_no_plus() { - assert_eq!( - Err(ModOffParseError::InvalidFormat), - ModOff::parse("foo.exe4141") - ); - } - #[test] - fn parse_bad_no_digits() { - assert_eq!( - Err(ModOffParseError::InvalidFormat), - ModOff::parse("foo.exe+") - ); - } - - #[test] - fn parse_bad_bad_digits() { - assert_eq!( - Err(ModOffParseError::InvalidFormat), - ModOff::parse("foo.exe+41zz") - ); - } -} diff --git a/src/agent/srcview/src/pdbcache.rs b/src/agent/srcview/src/pdbcache.rs deleted file mode 100644 index 05dd947ce5..0000000000 --- a/src/agent/srcview/src/pdbcache.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::collections::BTreeMap; -use std::fs::File; -use std::path::{Path, PathBuf}; - -use anyhow::{format_err, Result}; -use log::*; -use pdb::{FallibleIterator, SymbolData, PDB}; -use serde::{Deserialize, Serialize}; - -use crate::SrcLine; - -#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] -pub struct PdbCache { - offset_to_line: BTreeMap, - symbol_to_lines: BTreeMap>, - path_to_symbols: BTreeMap>, - path_to_lines: BTreeMap>, -} - -impl PdbCache { - pub fn new>(pdb: P) -> Result { - let mut offset_to_line: BTreeMap = BTreeMap::new(); - let mut symbol_to_lines: BTreeMap> = BTreeMap::new(); - - // NOTE: We're using strings as the keys for now while we build the trees, since - // PathBuf comparisons are expensive. - let mut path_to_symbols: BTreeMap> = BTreeMap::new(); - let mut path_to_lines: BTreeMap> = BTreeMap::new(); - - let pdbfile = File::open(pdb)?; - let mut pdb = PDB::open(pdbfile)?; - - let address_map = pdb.address_map()?; - let string_table = pdb.string_table()?; - - let dbi = pdb.debug_information()?; - let mut modules = dbi.modules()?; - while let Some(module) = modules.next()? { - let info = match pdb.module_info(&module)? { - Some(info) => info, - None => { - warn!("no module info: {:?}", &module); - continue; - } - }; - - let program = info.line_program()?; - let mut symbols = info.symbols()?; - - while let Some(symbol) = symbols.next()? { - if let Ok(SymbolData::Procedure(proc)) = symbol.parse() { - let proc_name = proc.name.to_string(); - let mut lines = program.lines_for_symbol(proc.offset); - - let symbol_to_lines = symbol_to_lines.entry(proc_name.to_string()).or_default(); - - while let Some(line_info) = lines.next()? { - let rva = line_info - .offset - .to_rva(&address_map) - .ok_or_else(|| format_err!("invalid RVA: {:?}", line_info))?; - let file_info = program.get_file_info(line_info.file_index)?; - let file_name = file_info.name.to_string_lossy(&string_table)?; - - let path = file_name.into_owned(); - let line = line_info.line_start as usize; - - let srcloc = SrcLine::new(path.clone(), line); - - offset_to_line.insert(rva.0 as usize, srcloc.clone()); - path_to_symbols - .entry(path.clone()) - .or_default() - .push(proc_name.to_string()); - symbol_to_lines.push(srcloc.clone()); - path_to_lines.entry(path.clone()).or_default().push(line); - } - } - } - } - - Ok(Self { - offset_to_line, - symbol_to_lines, - path_to_symbols: path_to_symbols - .into_iter() - .map(|(p, s)| (PathBuf::from(p), s)) - .collect(), - path_to_lines: path_to_lines - .into_iter() - .map(|(p, l)| (PathBuf::from(p), l)) - .collect(), - }) - } - - pub fn offset(&self, off: &usize) -> Option<&SrcLine> { - self.offset_to_line.get(off) - } - - pub fn paths(&self) -> impl Iterator { - self.path_to_lines.keys() - } - - pub fn path_symbols>(&self, path: P) -> Option> { - self.path_to_symbols - .get(path.as_ref()) - .map(|x| x.iter().map(|y| y.as_str())) - } - - pub fn path_lines>(&self, path: P) -> Option> { - self.path_to_lines.get(path.as_ref()).map(|x| x.iter()) - } - - pub fn symbol(&self, sym: &str) -> Option> { - self.symbol_to_lines.get(sym).map(|x| x.iter()) - } -} diff --git a/src/agent/srcview/src/report.rs b/src/agent/srcview/src/report.rs deleted file mode 100644 index 4ba478e744..0000000000 --- a/src/agent/srcview/src/report.rs +++ /dev/null @@ -1,570 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use anyhow::{format_err, Context, Result}; -use log::warn; -use regex::Regex; - -use crate::{SrcLine, SrcView}; - -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -struct FileCov { - symbols: BTreeMap>, - lines: Vec, - hits: Vec, -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -struct DirCov { - hits: usize, - lines: usize, -} - -impl DirCov { - fn new(hits: usize, lines: usize) -> Self { - Self { hits, lines } - } -} - -/// A path keyed structure to store both coverage and source info such that we can easily emit a -/// serialized representation of the source and its coverage -pub struct Report { - filecov: BTreeMap, - dircov: BTreeMap, - overall: DirCov, -} - -impl Report { - /// Create a new report from coverage, a `SrcView`, and an optional regex - /// - /// # Arguments - /// - /// * `coverage` - The hit set of SrcLines - /// * `srcview` - The total set of SrcLines - /// * `include_regex` - A regular expression that will be applied against the file - /// paths from the srcview. Any files matching this regex will be - /// included in the output report. This is to exclude dependencies - /// (e.g. the CRT) and should frequently be your project root. E.g. - /// if you have git repo 'Foo' that was built on z:\src\Foo, a - /// reasonable regex might be r'z:\\src\\Foo' (note that is is a - /// raw string and the backslashes are to escape backreferences in - /// the regex). A of `None` matches all files and includes them. - /// - /// # Errors - /// - /// If the regex cannot be compiled - /// - /// # Example - /// ```no_run - /// use srcview::{ModOff, Report, SrcLine, SrcView}; - /// - /// let modoff_data = std::fs::read_to_string("coverage.modoff.txt").unwrap(); - /// let modoffs = ModOff::parse(&modoff_data).unwrap(); - /// - /// let mut srcview = SrcView::new(); - /// srcview.insert("example.exe", "example.pdb").unwrap(); - /// - /// let coverage: Vec = modoffs - /// .into_iter() - /// .filter_map(|m| srcview.modoff(&m)) - /// .collect(); - /// - /// let r = Report::new(&coverage, &srcview, Some(r"E:\\1f\\coverage\\example")).unwrap(); - /// ``` - pub fn new( - coverage: &[SrcLine], - srcview: &SrcView, - include_regex: Option<&str>, - ) -> Result { - let include = include_regex.map(Regex::new).transpose()?; - let filecov = Self::compute_filecov(coverage, srcview, &include)?; - - // should this function take &[ModOff] and perform the conversion itself? - - let mut r = Self { - filecov, - // these will be populated by compute_dircov - dircov: BTreeMap::new(), - overall: DirCov::new(0, 0), - }; - - r.compute_dircov(); - - Ok(r) - } - - // should only be called from new, function to initalize file coverage - fn compute_filecov( - coverage: &[SrcLine], - srcview: &SrcView, - include: &Option, - ) -> Result> { - let uniq_cov: BTreeSet = coverage.iter().cloned().collect(); - - let mut filecov = BTreeMap::new(); - - for path in srcview.paths() { - if !Self::relevant_path(path, include)? { - continue; - } - - let path_srclocs = srcview - .path_lines(path) - .ok_or_else(|| { - format_err!("unable to find path lines in path: {}", path.display()) - })? - .map(|line| SrcLine::new(path, line)); - - let mut lines = vec![]; - let mut hits = vec![]; - let mut symbols = BTreeMap::new(); - - for srcloc in path_srclocs { - lines.push(srcloc.line); - - if uniq_cov.contains(&srcloc) { - hits.push(srcloc.line); - } - } - - lines.sort_unstable(); - hits.sort_unstable(); - - if let Some(path_symbols) = srcview.path_symbols(path) { - for symbol in path_symbols { - let symbol_srclocs: BTreeSet = srcview - .symbol(&symbol) - .ok_or_else(|| format_err!("unable to resolve symbol: {}", symbol))? - .cloned() - .collect(); - - symbols.insert(symbol, symbol_srclocs); - } - } - - filecov.insert( - path.clone(), - FileCov { - lines, - hits, - symbols, - }, - ); - } - - Ok(filecov) - } - - // should only be called from `new`, function to initialize directory coverage and overall - // coverage. File coverage must be already initialized at this point - fn compute_dircov(&mut self) { - // need to make a copy so we don't hold an immutable reference to self in the loop - let paths: Vec = self.paths().cloned().collect(); - - // on windows we can have multiple roots, e.g. c:\ and z:\. Thus we need to track all - // the roots we see and their coverage to total them later for overall project coverage - let mut overall: BTreeMap = BTreeMap::new(); - - for path in paths { - let mut anc = path.ancestors(); - - // the /foo/bar/baz.c - discard baz.c - let _ = anc.next(); - - // for the rest of the directories, lets compute them - for dir in anc { - // if we've already computed it, were good - if self.dir(dir).is_some() { - continue; - } - - let mut hits = 0; - let mut lines = 0; - - // get every file that matches this directory and total it - for file in self.filter_files(dir) { - if let Some(cov) = self.file(file) { - hits += cov.hits.len(); - lines += cov.lines.len(); - } - } - - self.dircov - .insert(dir.to_path_buf(), DirCov::new(hits, lines)); - } - - // now lets get the root so we can compute the overall stats - let anc = path.ancestors(); - - if let Some(root) = anc.last() { - // at this point we know we've computed this - match self.dircov.get(root) { - Some(dircov) => { - // we don't really care if we're overwriting it - overall.insert(root.to_path_buf(), *dircov); - } - None => { - warn!( - "unable to get root for path for directory stats. root: {}", - root.display() - ); - } - } - } - } - - let mut total = DirCov::new(0, 0); - - for dircov in overall.values() { - total.hits += dircov.hits; - total.lines += dircov.lines; - } - - self.overall = total; - } - - fn file>(&self, path: P) -> Option<&FileCov> { - self.filecov.get(path.as_ref()) - } - - fn dir>(&self, path: P) -> Option<&DirCov> { - self.dircov.get(path.as_ref()) - } - - fn paths(&self) -> impl Iterator { - self.filecov.keys() - } - - fn dirs(&self) -> impl Iterator { - self.dircov.keys() - } - - fn filter_files>(&self, path: P) -> impl Iterator { - self.paths().filter(move |p| p.starts_with(path.as_ref())) - } - - fn dir_has_files>(&self, path: P) -> bool { - for file in self.filter_files(path.as_ref()) { - let mut anc = file.ancestors(); - let _ = anc.next(); - if let Some(dir) = anc.next() { - if dir == path.as_ref() { - return true; - } - } - } - - false - } - - // wrapper to allow ergonomic filtering with an option - fn filter_path + fmt::Debug>( - path: P, - filter: &Option, - ) -> Result { - match filter { - Some(regex) => { - // we need our path as a string to regex it - let path_string = path.as_ref().to_str().ok_or_else(|| { - format_err!("could not utf8 decode path: {}", path.as_ref().display()) - })?; - - let filtered = regex.replace(path_string, "").into_owned(); - - Ok(PathBuf::from(filtered)) - } - None => Ok(path.as_ref().to_path_buf()), - } - } - - // wrapper to allow ergonomic testing of our include regex inside an option against a - // path - fn relevant_path + fmt::Debug>( - path: P, - include: &Option, - ) -> Result { - match include { - Some(regex) => { - // we need our path as a string to regex it - let path_string = path.as_ref().to_str().ok_or_else(|| { - format_err!("could not utf8 decode path: {}", path.as_ref().display()) - })?; - - Ok(regex.is_match(path_string)) - } - None => Ok(true), - } - } - - /// Generate a Cobertura report - /// - /// # Arguments - /// - /// * `filter_regex` - This a search and replace regex that is applied to all file - /// paths that will appear in the output report. This is specifically - /// useful as many coverage visualization tools will require paths to - /// match, and by default debug paths include the build machine info. - /// For example, if our repo is 'Foo' and has `test.c` in it, the - /// debug path could be `z:\build\Foo\test.c`. In the generated report - /// we would want to strip off the build info and the repo name, such - /// that the path that remains is relative to the repo root. As a - /// result we might pass r"z:\\build\Foo\\". When applied to our SrcView - /// paths this will replace that regex with the empty string, leaving the - /// path `test.c` which relative to our repo root is correct. A value of - /// `None` will not filter any paths. - /// - /// # Errors - /// - /// * If the filter regex cannot be compiled - /// * If there is an error writing the output xml - /// - /// # Example - /// - /// ```no_run - /// use srcview::{ModOff, Report, SrcLine, SrcView}; - /// - /// let modoff_data = std::fs::read_to_string("coverage.modoff.txt").unwrap(); - /// let modoffs = ModOff::parse(&modoff_data).unwrap(); - /// - /// let mut srcview = SrcView::new(); - /// srcview.insert("example.exe", "example.pdb").unwrap(); - /// - /// let coverage: Vec = modoffs - /// .into_iter() - /// .filter_map(|m| srcview.modoff(&m)) - /// .collect(); - /// - /// // in this case our repo is `coverage`, and has an `example` directory containing - /// // our code files. Anything that matches this path shoudl be included. - /// let r = Report::new(&coverage, &srcview, Some(r"E:\\1f\\coverage\\example")).unwrap(); - /// - /// // NOTE: If you're writing out to a file, you'll almost certainly want to wrap it - /// // in a `BufWriter` before passing it into `cobertura()`. - /// let mut xml = Vec::new(); - /// - /// // However when generating the report, we want to strip off only the repo name -- - /// // `example` is inside the repo so to make the paths line up we need to leave it. - /// r.cobertura(Some(r"E:\\1f\coverage\\"), &mut xml).unwrap(); - /// - /// println!("{}", std::str::from_utf8(&xml).unwrap()); - /// ``` - pub fn cobertura(&self, filter_regex: Option<&str>, output: &mut W) -> Result<()> { - use quick_xml::{ - events::{BytesEnd, BytesStart, BytesText, Event}, - Writer, - }; - - let filter = filter_regex.map(Regex::new).transpose()?; - - // HACK(ish): Allocate a single XML start element and reuse it throughout. - // - // We do this since adding attributes to `BytesStart` will potentially cause its - // internal buffer to be expanded, which incurs a reallocation. Instead of doing - // this multiple hundreds or thousands of times, we can simply reuse a single - // buffer and clear it inbetween (which does not shrink the underlying buffer). - let mut el_start = BytesStart::new(""); - - let mut ew = Writer::new_with_indent(output, b' ', 2); - - // xml-rs does not support DTD entries yet, but thankfully ADO's parser is loose - - let unixtime = SystemTime::now() - .duration_since(UNIX_EPOCH) - .context("system time before unix epoch")? - .as_secs(); - - ew.write_event(Event::Start( - BytesStart::new("coverage").with_attributes([ - ("lines-valid", format!("{}", self.overall.lines).as_str()), - ("lines-covered", format!("{}", self.overall.hits).as_str()), - ( - "line-rate", - format!( - "{:.02}", - self.overall.hits as f32 / self.overall.lines as f32 - ) - .as_str(), - ), - ("branches-valid", "0"), - ("branches-covered", "0"), - ("branch-rate", "0"), - ("timestamp", format!("{unixtime}").as_str()), - ("complexity", "0"), - ("version", "0.1"), - ]), - ))?; - - ew.create_element("sources").write_inner_content(|ew| { - ew.create_element("source") - .write_text_content(BytesText::new(""))?; - - Ok(()) - })?; - - ew.write_event(Event::Start(BytesStart::new("packages")))?; - - for dir in self.dirs() { - if !self.dir_has_files(dir) { - continue; - } - - let display_dir = Self::filter_path(dir, &filter)?.display().to_string(); - - ew.write_event(Event::Start( - el_start - .clear_attributes() - .set_name(b"package") - .extend_attributes([("name", display_dir.as_str())]) - .borrow(), - ))?; - ew.write_event(Event::Start(BytesStart::new("classes")))?; - - // - // PER-FILE - // - - for path in self.filter_files(dir) { - let display_path = Self::filter_path(path, &filter)?.display().to_string(); - - let filecov = match self.file(path) { - Some(filecov) => filecov, - None => { - warn!("unable to find coverage for path: {}", path.display()); - continue; - } - }; - - let file_srclocs: BTreeSet = filecov - .lines - .iter() - .map(|line| SrcLine::new(path, *line)) - .collect(); - let hit_srclocs: BTreeSet = filecov - .hits - .iter() - .map(|line| SrcLine::new(path, *line)) - .collect(); - - ew.write_event(Event::Start( - el_start - .clear_attributes() - .set_name(b"class") - .extend_attributes([ - ("name", display_path.as_str()), - ("filename", display_path.as_str()), - ( - "line-rate", - format!( - "{:.02}", - filecov.hits.len() as f32 / filecov.lines.len() as f32 - ) - .as_str(), - ), - ("branch-rate", "0"), - ]) - .borrow(), - ))?; - - // - // METHODS - // - - ew.write_event(Event::Start(BytesStart::new("methods")))?; - - for (symbol, symbol_srclocs) in filecov.symbols.iter() { - let mut symbol_hits = 0; - for hit in &hit_srclocs { - if symbol_srclocs.contains(hit) { - symbol_hits += 1; - } - } - - ew.write_event(Event::Start( - el_start - .clear_attributes() - .set_name(b"method") - .extend_attributes([ - ("name", symbol.as_str()), - ("signature", ""), - ( - "line-rate", - format!( - "{:.02}", - symbol_hits as f32 / symbol_srclocs.len() as f32 - ) - .as_str(), - ), - ("branch-rate", "0"), - ]) - .borrow(), - ))?; - - ew.write_event(Event::Start(BytesStart::new("lines")))?; - - for srcloc in symbol_srclocs { - let hits = if hit_srclocs.contains(srcloc) { - "1" - } else { - "0" - }; - - ew.write_event(Event::Empty( - el_start - .clear_attributes() - .set_name(b"line") - .extend_attributes([ - ("number", format!("{}", srcloc.line).as_str()), - ("hits", hits), - ("branch", "false"), - ]) - .borrow(), - ))?; - } - - ew.write_event(Event::End(BytesEnd::new("lines")))?; - ew.write_event(Event::End(BytesEnd::new("method")))?; - } - ew.write_event(Event::End(BytesEnd::new("methods")))?; - - ew.write_event(Event::Start(BytesStart::new("lines")))?; - - // - // LINES - // - for srcloc in &file_srclocs { - let hits = if hit_srclocs.contains(srcloc) { - "1" - } else { - "0" - }; - - ew.write_event(Event::Empty( - el_start - .clear_attributes() - .set_name(b"line") - .extend_attributes([ - ("number", format!("{}", srcloc.line).as_str()), - ("hits", hits), - ("branch", "false"), - ]) - .borrow(), - ))?; - } - ew.write_event(Event::End(BytesEnd::new("lines")))?; - ew.write_event(Event::End(BytesEnd::new("class")))?; - } - ew.write_event(Event::End(BytesEnd::new("classes")))?; - ew.write_event(Event::End(BytesEnd::new("package")))?; - } - ew.write_event(Event::End(BytesEnd::new("packages")))?; - ew.write_event(Event::End(BytesEnd::new("coverage")))?; - - Ok(()) - } -} diff --git a/src/agent/srcview/src/srcline.rs b/src/agent/srcview/src/srcline.rs deleted file mode 100644 index da75f24563..0000000000 --- a/src/agent/srcview/src/srcline.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::cmp::Ordering; -use std::fmt; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; - -/// A path and a line number -#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] -pub struct SrcLine { - pub path: PathBuf, - pub line: usize, -} - -impl fmt::Display for SrcLine { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "{}:{}", &self.path.display(), self.line) - } -} - -impl Ord for SrcLine { - fn cmp(&self, other: &Self) -> Ordering { - let path_cmp = self.path.cmp(&other.path); - - if path_cmp != Ordering::Equal { - return path_cmp; - } - - self.line.cmp(&other.line) - } -} - -impl PartialOrd for SrcLine { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl SrcLine { - pub fn new>(path: P, line: usize) -> Self { - let path = path.as_ref().to_owned(); - Self { path, line } - } -} diff --git a/src/agent/srcview/src/srcview.rs b/src/agent/srcview/src/srcview.rs deleted file mode 100644 index b976e5730d..0000000000 --- a/src/agent/srcview/src/srcview.rs +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; - -use anyhow::Result; -use serde::{Deserialize, Serialize}; - -use crate::{ModOff, PdbCache, SrcLine}; - -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub struct SrcView(BTreeMap); - -/// A SrcView is a collection of zero or more PdbCaches for easy querying. It stores all -/// the mapping information from the PDBs. It does _not_ contain any coverage information. -impl SrcView { - /// Create's a new SrcView - pub fn new() -> Self { - Self::default() - } - - /// Insert a new pdb into the SrcView, returning any previous pdb info that you're - /// replacing. In most cases this return value can be ignored. - /// - /// # Arguments - /// - /// * `module` - Module name to store the PDB info as - /// * `pdb` - Path to PDB - /// - /// # Errors - /// - /// If the PDB at the provided path cannot be parsed or contains otherwise unexpected data. - /// - /// # Example - /// - /// ```no_run - /// use srcview::SrcView; - /// - /// let mut sv = SrcView::new(); - /// - /// // Map the contents of 'example.pdb' to the module name 'example.exe' - /// sv.insert("example.exe", r"z:\src\example.pdb").unwrap(); - /// - /// // you can now query sv for info from example.exe... - /// ``` - pub fn insert>(&mut self, module: &str, pdb: P) -> Result> { - let cache = PdbCache::new(pdb)?; - Ok(self.0.insert(module.to_owned(), cache)) - } - - /// Insert a new pdb into the SrcView only if the `pdb` path is not in the SrcView already, - /// returning a [Result] indicating the success of the insert, if any was necessary. - /// If the [Result] is [Ok], the contained bool indicates whether a value was inserted. - /// - /// # Arguments - /// - /// * `module` - Module name to store the PDB info as - /// * `pdb` - Path to PDB - /// - /// # Errors - /// - /// If the PDB at the provided path has not been inserted already **and** - /// cannot be parsed or contains otherwise unexpected data. - /// - /// # Example - /// - /// ```ignore - /// use srcview::SrcView; - /// - /// let mut sv = SrcView::new(); - /// let modoffs = get_coverage(); - /// - /// // Map each modoff to a PDB name/path and make sure it's in the SrcView - /// for modoff in modoffs { - /// let module_name = mod_name_from_modoff(modoff); - /// let res = sv.try_insert(module_name, format!("~/pdbs/{module_name}.pdb")); - /// - /// if let Ok(inserted) = res { - /// println!("PDB was inserted: {inserted}"); - /// } - /// } - /// ``` - pub fn try_insert>(&mut self, module: &str, pdb: P) -> Result { - if self.0.contains_key(&module.to_owned()) { - Ok(false) - } else { - self.insert(module, pdb).map(|_| true) - } - } - - /// Resolve a modoff to SrcLine, if one exists - /// - /// # Arguments - /// - /// * `modoff` - Reference to a ModOff you'd like to resolve - /// - /// # Example - /// - /// ```no_run - /// use srcview::{ModOff, SrcView}; - /// - /// let mut sv = SrcView::new(); - /// - /// // Map the contents of 'example.pdb' to the module name 'example.exe' - /// sv.insert("example.exe", r"z:\src\example.pdb").unwrap(); - /// - /// let modoff = ModOff::new("example.exe", 0x4141); - /// - /// if let Some(srcline) = sv.modoff(&modoff) { - /// println!("Resolved {} to {}", modoff, srcline); - /// } - /// ``` - pub fn modoff(&self, modoff: &ModOff) -> Option { - match self.0.get(&modoff.module) { - Some(cache) => cache.offset(&modoff.offset).cloned(), - None => None, - } - } - - /// Resolve a symbol (e.g. module!name) to its possible SrcLines, if such a symbol - /// exists - /// - /// # Arguments - /// - /// * `sym` - A symbol name you'd like to query in the form of `!` - /// - /// # Example - /// - /// ```no_run - /// use srcview::SrcView; - /// - /// let mut sv = SrcView::new(); - /// - /// // Map the contents of 'example.pdb' to the module name 'example.exe' - /// sv.insert("example.exe", r"z:\src\example.pdb").unwrap(); - /// - /// if let Some(srclines) = sv.symbol("example.exe!main") { - /// println!("example.exe!main has the following valid source lines:"); - /// for line in srclines { - /// println!(" - {}", line); - /// } - /// }; - /// ``` - pub fn symbol(&self, sym: &str) -> Option> { - let split: Vec<&str> = sym.split('!').collect(); - - if split.len() < 2 { - return None; - } - - let module = split[0]; - let name: String = split[1..].join("!"); - - match self.0.get(module) { - Some(cache) => cache.symbol(&name), - None => None, - } - } - - /// Resolve a path to its possible SrcLines, if such a path exists - /// - /// # Arguments - /// - /// * `path` - An absolute path that possibly matches one from the debug info - /// - /// # Example - /// - /// ```no_run - /// use srcview::SrcView; - /// - /// let mut sv = SrcView::new(); - /// - /// // Map the contents of 'example.pdb' to the module name 'example.exe' - /// sv.insert("example.exe", r"z:\src\example.pdb").unwrap(); - /// - /// if let Some(srclines) = sv.path_lines(r"z:\src\example.c") { - /// println!("example.c has the following valid source lines:"); - /// for line in srclines { - /// println!(" - {}", line); - /// } - /// } - /// ``` - pub fn path_lines>(&self, path: P) -> Option> { - // we want to unique the lines in use across all loaded pdbs - let mut r: BTreeSet = BTreeSet::new(); - - for cache in self.0.values() { - if let Some(lines) = cache.path_lines(path.as_ref()) { - for line in lines { - r.insert(*line); - } - } - } - - if r.is_empty() { - return None; - } - - // convert to a vec - let mut v: Vec = r.into_iter().collect(); - - // lists of line numbers are nicer when theyre sorted... - v.sort_unstable(); - - Some(v.into_iter()) - } - - /// Resolve a path to its possible symbols, if such a path exists - /// - /// # Arguments - /// - /// * `path` - An absolute path that possibly matches one from the debug info - /// - /// # Example - /// - /// ```no_run - /// use srcview::SrcView; - /// - /// let mut sv = SrcView::new(); - /// - /// // Map the contents of 'example.pdb' to the module name 'example.exe' - /// sv.insert("example.exe", r"z:\src\example.pdb").unwrap(); - /// - /// if let Some(symbols) = sv.path_symbols(r"z:\src\example.c") { - /// println!("example.c has the following valid symbols:"); - /// for line in symbols { - /// println!(" - {}", line); - /// } - /// } - /// ``` - pub fn path_symbols>(&self, path: P) -> Option> { - // we want to unique the lines in use across all loaded pdbs - let mut r: BTreeSet = BTreeSet::new(); - - for (module, cache) in self.0.iter() { - if let Some(symbols) = cache.path_symbols(path.as_ref()) { - for sym in symbols { - r.insert(format!("{module}!{sym}")); - } - } - } - - if r.is_empty() { - return None; - } - - // convert to a vec - let mut v: Vec = r.into_iter().collect(); - - // lists of symbols are nicer when theyre sorted... - v.sort(); - - Some(v.into_iter()) - } - - /// Returns an iterator over all paths in the SrcView - /// - /// # Example - /// - /// ```no_run - /// use srcview::SrcView; - /// - /// let mut sv = SrcView::new(); - /// - /// // Map the contents of 'example.pdb' to the module name 'example.exe' - /// sv.insert("example.exe", r"z:\src\example.pdb").unwrap(); - /// - /// println!("paths in example.pdb:"); - /// - /// for path in sv.paths() { - /// println!(" - {}", path.display()); - /// } - /// ``` - pub fn paths(&self) -> impl Iterator { - let mut r: BTreeSet<&PathBuf> = BTreeSet::new(); - - for cache in self.0.values() { - for pb in cache.paths() { - r.insert(pb); - } - } - - r.into_iter() - } -} diff --git a/src/agent/srcview/tests/modoff.rs b/src/agent/srcview/tests/modoff.rs deleted file mode 100644 index 49b5890dee..0000000000 --- a/src/agent/srcview/tests/modoff.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::path::PathBuf; -use std::{env, fs}; - -use srcview::ModOff; - -#[test] -fn parse_modoff() { - let root = env::var("CARGO_MANIFEST_DIR").unwrap(); - let path: PathBuf = [&root, "res", "example.txt"].iter().collect(); - - let modoff = fs::read_to_string(path).unwrap(); - let modoffs = ModOff::parse(&modoff).unwrap(); - - assert_eq!( - modoffs, - vec![ - ModOff::new("example.exe", 0x6f70), - ModOff::new("example.exe", 0x6f75), - ModOff::new("example.exe", 0x6f79), - ModOff::new("example.exe", 0x6f7d), - ModOff::new("example.exe", 0x6f81), - ModOff::new("example.exe", 0x6f82), - ModOff::new("example.exe", 0x6f85), - ModOff::new("example.exe", 0x6f87), - ModOff::new("example.exe", 0x6f89), - ModOff::new("example.exe", 0x6f8b), - ModOff::new("example.exe", 0x6f9b), - ModOff::new("example.exe", 0x6fa2), - ModOff::new("example.exe", 0x6fa7), - ModOff::new("example.exe", 0x6fa9), - ModOff::new("example.exe", 0x6fad), - ] - ); -} diff --git a/src/agent/srcview/tests/srcview.rs b/src/agent/srcview/tests/srcview.rs deleted file mode 100644 index 29ba90f64f..0000000000 --- a/src/agent/srcview/tests/srcview.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -// tests depends on example.pdb -// $ sha256sum example.pdb -// ecc4214d687c97e9c8afd0c84b4b75383eaa0a237f8a8ca5049478f63b2c98b9 example.pdb - -use std::env; -use std::path::PathBuf; - -use srcview::{ModOff, SrcLine, SrcView}; - -fn test_srcview() -> SrcView { - let root = env::var("CARGO_MANIFEST_DIR").unwrap(); - let pdb_path: PathBuf = [&root, "res", "example.pdb"].iter().collect(); - - let mut srcview = SrcView::new(); - srcview.insert("example.exe", pdb_path).unwrap(); - - srcview -} - -#[test] -#[cfg_attr(not(feature = "binary-tests"), ignore)] -fn modoff() { - let srcview = test_srcview(); - - let good_modoff = ModOff::new("example.exe", 0x6f70); - assert_eq!( - srcview.modoff(&good_modoff), - Some(SrcLine::new("E:\\1f\\coverage\\example\\example.c", 3)) - ); - - let bad_offset = ModOff::new("example.exe", 0x4141); - assert_eq!(srcview.modoff(&bad_offset), None); - - let bad_module = ModOff::new("foo.exe", 0x4141); - assert_eq!(srcview.modoff(&bad_module), None); -} - -#[test] -#[cfg_attr(not(feature = "binary-tests"), ignore)] -fn symbol() { - let srcview = test_srcview(); - - let good: Vec<&SrcLine> = srcview.symbol("example.exe!main").unwrap().collect(); - - assert_eq!( - good, - vec![ - &SrcLine::new("E:\\1f\\coverage\\example\\example.c", 3), - &SrcLine::new("E:\\1f\\coverage\\example\\example.c", 4), - &SrcLine::new("E:\\1f\\coverage\\example\\example.c", 5), - &SrcLine::new("E:\\1f\\coverage\\example\\example.c", 6), - &SrcLine::new("E:\\1f\\coverage\\example\\example.c", 7), - &SrcLine::new("E:\\1f\\coverage\\example\\example.c", 10), - &SrcLine::new("E:\\1f\\coverage\\example\\example.c", 11), - ] - ); - - assert!(srcview.symbol("dosenotexist").is_none()); -} - -#[test] -#[cfg_attr(not(feature = "binary-tests"), ignore)] -fn path() { - let srcview = test_srcview(); - - let good: Vec = srcview - .path_lines("E:\\1f\\coverage\\example\\example.c") - .unwrap() - .collect(); - - assert_eq!(good, vec![3, 4, 5, 6, 7, 10, 11]); - - assert!(srcview.path_lines("z:\\does\\not\\exist.c").is_none()); -} diff --git a/src/ci/agent.sh b/src/ci/agent.sh index 226cf02eae..092d1c5a72 100755 --- a/src/ci/agent.sh +++ b/src/ci/agent.sh @@ -54,19 +54,16 @@ if [ "$(uname)" == 'Linux' ]; then # see: https://github.com/benesch/materialize/blob/02cc34cb7c7d9f1b775fe95e1697b797d8fa9b6d/misc/python/mzbuild.py#L166-L183 objcopy -R .debug_pubnames -R .debug_pubtypes target/release/onefuzz-task objcopy -R .debug_pubnames -R .debug_pubtypes target/release/onefuzz-agent - objcopy -R .debug_pubnames -R .debug_pubtypes target/release/srcview echo "Compressing debug symbols" objcopy --compress-debug-sections target/release/onefuzz-task objcopy --compress-debug-sections target/release/onefuzz-agent - objcopy --compress-debug-sections target/release/srcview fi echo "Copying artifacts to $output_dir" cp target/release/onefuzz-task* "$output_dir" cp target/release/onefuzz-agent* "$output_dir" -cp target/release/srcview* "$output_dir" if exists target/release/*.pdb; then for file in target/release/*.pdb; do diff --git a/src/ci/check-dependencies.sh b/src/ci/check-dependencies.sh index 0917e0af31..79f21845c1 100755 --- a/src/ci/check-dependencies.sh +++ b/src/ci/check-dependencies.sh @@ -127,28 +127,3 @@ secur32.dll sspicli.dll ucrtbase.dll ws2_32.dll" - -check "$script_dir/../agent/target/release/srcview" \ -"/lib64/ld-linux-x86-64.so.2 -libc.so.6 -libgcc_s.so.1 -libpthread.so.0 -linux-vdso.so.1" \ -\ -"/lib/ld-linux-aarch64.so.1 -libc.so.6 -libgcc_s.so.1 -libpthread.so.0 -linux-vdso.so.1" \ -\ -"advapi32.dll -apphelp.dll -bcrypt.dll -bcryptprimitives.dll -cryptbase.dll -kernel32.dll -kernelbase.dll -msvcrt.dll -ntdll.dll -rpcrt4.dll -sechost.dll"