From 9ec4f74fcb9fd26477cbf3be1578dfb0dead2793 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 24 Mar 2021 11:16:08 -0700 Subject: [PATCH 1/7] wiring all syncdirs to the UI to monitor their file count Added new section for the coverage in the UI Added event broadcast in the the telemtry lib Gracefully exit from the UI --- src/agent/Cargo.lock | 1 + src/agent/onefuzz-agent/src/local/cmd.rs | 24 +- src/agent/onefuzz-agent/src/local/common.rs | 73 ++--- .../src/local/generic_analysis.rs | 39 ++- .../src/local/generic_crash_report.rs | 33 +- .../src/local/generic_generator.rs | 28 +- .../onefuzz-agent/src/local/libfuzzer.rs | 56 +--- .../src/local/libfuzzer_coverage.rs | 26 +- .../src/local/libfuzzer_crash_report.rs | 36 ++- .../onefuzz-agent/src/local/libfuzzer_fuzz.rs | 25 +- .../src/local/libfuzzer_merge.rs | 22 +- .../src/local/libfuzzer_regression.rs | 10 +- .../src/local/libfuzzer_test_input.rs | 10 +- src/agent/onefuzz-agent/src/local/radamsa.rs | 14 +- .../onefuzz-agent/src/local/test_input.rs | 10 +- src/agent/onefuzz-agent/src/local/tui.rs | 284 +++++++++++++++--- .../src/tasks/fuzz/libfuzzer_fuzz.rs | 6 +- src/agent/onefuzz-telemetry/Cargo.toml | 4 +- src/agent/onefuzz-telemetry/src/lib.rs | 25 ++ 19 files changed, 516 insertions(+), 210 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 62291ddb25..6e5fc87f1c 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -1811,6 +1811,7 @@ version = "0.1.0" dependencies = [ "appinsights", "iced-x86", + "lazy_static", "log", "serde", "tokio 0.2.25", diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs index d4ecced5dd..12382c276a 100644 --- a/src/agent/onefuzz-agent/src/local/cmd.rs +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -39,18 +39,20 @@ pub async fn run(args: clap::ArgMatches<'static>) -> Result<()> { let event_sender = terminal.as_ref().map(|t| t.task_events.clone()); let command_run = tokio::spawn(async move { match args.subcommand() { - (RADAMSA, Some(sub)) => radamsa::run(sub).await, + (RADAMSA, Some(sub)) => radamsa::run(sub, event_sender).await, (LIBFUZZER, Some(sub)) => libfuzzer::run(sub, event_sender).await, - (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, - (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, - (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, - (LIBFUZZER_MERGE, Some(sub)) => libfuzzer_merge::run(sub).await, - (GENERIC_ANALYSIS, Some(sub)) => generic_analysis::run(sub).await, - (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub).await, - (GENERIC_GENERATOR, Some(sub)) => generic_generator::run(sub).await, - (GENERIC_TEST_INPUT, Some(sub)) => test_input::run(sub).await, - (LIBFUZZER_TEST_INPUT, Some(sub)) => libfuzzer_test_input::run(sub).await, - (LIBFUZZER_REGRESSION, Some(sub)) => libfuzzer_regression::run(sub).await, + (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub, event_sender).await, + (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub, event_sender).await, + (LIBFUZZER_CRASH_REPORT, Some(sub)) => { + libfuzzer_crash_report::run(sub, event_sender).await + } + (LIBFUZZER_MERGE, Some(sub)) => libfuzzer_merge::run(sub, event_sender).await, + (GENERIC_ANALYSIS, Some(sub)) => generic_analysis::run(sub, event_sender).await, + (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub, event_sender).await, + (GENERIC_GENERATOR, Some(sub)) => generic_generator::run(sub, event_sender).await, + (GENERIC_TEST_INPUT, Some(sub)) => test_input::run(sub, event_sender).await, + (LIBFUZZER_TEST_INPUT, Some(sub)) => libfuzzer_test_input::run(sub, event_sender).await, + (LIBFUZZER_REGRESSION, Some(sub)) => libfuzzer_regression::run(sub, event_sender).await, _ => { anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); } diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 348e0a037b..e8b41c487a 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -15,8 +15,7 @@ use std::{ path::{Path, PathBuf}, time::Duration, }; -use tokio::stream::StreamExt; -use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::delay_for}; +use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; pub const SETUP_DIR: &str = "setup_dir"; @@ -69,6 +68,7 @@ pub enum CmdType { pub struct LocalContext { pub job_path: PathBuf, pub common_config: CommonConfig, + pub event_sender: Option>, } pub fn get_hash_map(args: &clap::ArgMatches<'_>, name: &str) -> Result> { @@ -208,11 +208,16 @@ pub fn get_synced_dir( }) } + // NOTE: generate_task_id is intended to change the default behavior for local // fuzzing tasks from generating random task id to using UUID::nil(). This // enables making the one-shot crash report generation, which isn't really a task, // consistent across multiple runs. -pub fn build_local_context(args: &ArgMatches<'_>, generate_task_id: bool) -> Result { +pub fn build_local_context( + args: &ArgMatches<'_>, + generate_task_id: bool, + event_sender: Option>, +) -> Result { let job_id = get_uuid("job_id", args).unwrap_or_else(|_| Uuid::nil()); let task_id = get_uuid("task_id", args).unwrap_or_else(|_| { if generate_task_id { @@ -250,6 +255,7 @@ pub fn build_local_context(args: &ArgMatches<'_>, generate_task_id: bool) -> Res Ok(LocalContext { job_path, common_config, + event_sender, }) } @@ -317,48 +323,31 @@ pub async fn wait_for_dir(path: impl AsRef) -> Result<()> { .await } -pub fn spawn_file_count_monitor( - dir: PathBuf, - sender: UnboundedSender, -) -> JoinHandle> { - tokio::spawn(async move { - wait_for_dir(&dir).await?; - - loop { - let mut rd = tokio::fs::read_dir(&dir).await?; - let mut count: usize = 0; - - while let Some(Ok(entry)) = rd.next().await { - if entry.path().is_file() { - count += 1; - } - } +#[derive(Debug)] +pub enum UiEvent { + MonitorDir(PathBuf), +} - if sender - .send(UiEvent::FileCount { - dir: dir.clone(), - count, - }) - .is_err() - { - return Ok(()); - } - delay_for(Duration::from_secs(5)).await; - } - }) +pub trait SyncCountDirMonitor { + fn monitor_count(self, event_sender: &Option>) -> Result; } -pub fn monitor_file_urls( - urls: &[Option>], - event_sender: UnboundedSender, -) -> Vec>> { - urls.iter() - .filter_map(|x| x.as_ref()) - .map(|path| spawn_file_count_monitor(path.as_ref().into(), event_sender.clone())) - .collect::>() +impl SyncCountDirMonitor for SyncedDir { + fn monitor_count(self, event_sender: &Option>) -> Result { + if let (Some(event_sender), Some(p)) = (event_sender, self.url.as_file_path()) { + event_sender.send(UiEvent::MonitorDir(p))?; + } + Ok(self) + } } -#[derive(Debug)] -pub enum UiEvent { - FileCount { dir: PathBuf, count: usize }, +impl SyncCountDirMonitor> for Option { + fn monitor_count(self, event_sender: &Option>) -> Result { + if let Some(sd) = self { + let sd = sd.monitor_count(event_sender)?; + Ok(Some(sd)) + } else { + Ok(self) + } + } } diff --git a/src/agent/onefuzz-agent/src/local/generic_analysis.rs b/src/agent/onefuzz-agent/src/local/generic_analysis.rs index ef21bcb9c8..a1dd17581a 100644 --- a/src/agent/onefuzz-agent/src/local/generic_analysis.rs +++ b/src/agent/onefuzz-agent/src/local/generic_analysis.rs @@ -4,8 +4,9 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_exe, get_hash_map, get_synced_dir, CmdType, - ANALYSIS_DIR, ANALYZER_ENV, ANALYZER_EXE, ANALYZER_OPTIONS, CRASHES_DIR, NO_REPRO_DIR, - REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TOOLS_DIR, UNIQUE_REPORTS_DIR, + SyncCountDirMonitor, ANALYSIS_DIR, ANALYZER_ENV, ANALYZER_EXE, ANALYZER_OPTIONS, + CRASHES_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TOOLS_DIR, + UNIQUE_REPORTS_DIR, }, tasks::{ analysis::generic::{run as run_analysis, Config}, @@ -15,11 +16,15 @@ use crate::{ use anyhow::Result; use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; +use tokio::sync::mpsc::UnboundedSender; + +use super::common::UiEvent; pub fn build_analysis_config( args: &clap::ArgMatches<'_>, input_queue: Option, common: CommonConfig, + event_sender: Option>, ) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_options = get_cmd_arg(CmdType::Target, args); @@ -27,18 +32,25 @@ pub fn build_analysis_config( let analyzer_exe = value_t!(args, ANALYZER_EXE, String)?; let analyzer_options = args.values_of_lossy(ANALYZER_OPTIONS).unwrap_or_default(); let analyzer_env = get_hash_map(args, ANALYZER_ENV)?; - let analysis = get_synced_dir(ANALYSIS_DIR, common.job_id, common.task_id, args)?; + let analysis = get_synced_dir(ANALYSIS_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let tools = get_synced_dir(TOOLS_DIR, common.job_id, common.task_id, args)?; let crashes = if input_queue.is_none() { - get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args).ok() + get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)? } else { None }; - - let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args).ok(); - let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args).ok(); - let unique_reports = - get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args).ok(); + let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; + let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; + let unique_reports = get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; let config = Config { target_exe, @@ -59,9 +71,12 @@ pub fn build_analysis_config( Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let config = build_analysis_config(args, None, context.common_config.clone())?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_analysis_config(args, None, context.common_config.clone(), event_sender)?; run_analysis(config).await } diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 56b10fdac7..22c100d2bd 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -4,9 +4,9 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_DEBUGGER, - DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, - TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + SyncCountDirMonitor, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, + DISABLE_CHECK_DEBUGGER, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, + TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::{ config::CommonConfig, @@ -16,11 +16,15 @@ use crate::{ use anyhow::Result; use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; +use tokio::sync::mpsc::UnboundedSender; + +use super::common::UiEvent; pub fn build_report_config( args: &clap::ArgMatches<'_>, input_queue: Option, common: CommonConfig, + event_sender: Option>, ) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; @@ -31,16 +35,22 @@ pub fn build_report_config( common.job_id, common.task_id, args, - )?); - let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args).ok(); - let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args).ok(); + )?) + .monitor_count(&event_sender)?; + let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; + let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; let unique_reports = Some(get_synced_dir( UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args, - )?); + )?) + .monitor_count(&event_sender)?; let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); @@ -70,9 +80,12 @@ pub fn build_report_config( Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let config = build_report_config(args, None, context.common_config.clone())?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_report_config(args, None, context.common_config.clone(), event_sender)?; ReportTask::new(config).managed_run().await } diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index 07dae5562d..6462b744b2 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -4,9 +4,10 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, - get_synced_dirs, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, - DISABLE_CHECK_DEBUGGER, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, READONLY_INPUTS, - RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, TOOLS_DIR, + get_synced_dirs, CmdType, SyncCountDirMonitor, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, + CRASHES_DIR, DISABLE_CHECK_DEBUGGER, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, + READONLY_INPUTS, RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, + TOOLS_DIR, }, tasks::{ config::CommonConfig, @@ -15,9 +16,17 @@ use crate::{ }; use anyhow::Result; use clap::{App, Arg, SubCommand}; +use tokio::sync::mpsc::UnboundedSender; -pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> Result { - let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?; +use super::common::UiEvent; + +pub fn build_fuzz_config( + args: &clap::ArgMatches<'_>, + common: CommonConfig, + event_sender: Option>, +) -> Result { + let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_options = get_cmd_arg(CmdType::Target, args); let target_env = get_cmd_env(CmdType::Target, args)?; @@ -59,9 +68,12 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> R Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let config = build_fuzz_config(args, context.common_config.clone())?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?; GeneratorTask::new(config).run().await } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index 1a811e2b99..cb519fb235 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -4,8 +4,8 @@ use crate::{ local::{ common::{ - build_local_context, monitor_file_urls, wait_for_dir, DirectoryMonitorQueue, - ANALYZER_EXE, COVERAGE_DIR, REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR, + build_local_context, wait_for_dir, DirectoryMonitorQueue, ANALYZER_EXE, COVERAGE_DIR, + REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR, }, generic_analysis::{build_analysis_config, build_shared_args as build_analysis_args}, libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args}, @@ -34,18 +34,8 @@ pub async fn run( args: &clap::ArgMatches<'_>, event_sender: Option>, ) -> Result<()> { - let mut task_handles = vec![]; - let context = build_local_context(args, true)?; - let fuzz_config = build_fuzz_config(args, context.common_config.clone())?; - if let Some(event_sender) = event_sender.clone() { - task_handles.append(&mut monitor_file_urls( - &[ - fuzz_config.crashes.url.as_file_path(), - fuzz_config.inputs.url.as_file_path(), - ], - event_sender, - )); - } + let context = build_local_context(args, true, event_sender.clone())?; + let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?; let crash_dir = fuzz_config .crashes .url @@ -72,26 +62,8 @@ pub async fn run( task_id: Uuid::new_v4(), ..context.common_config.clone() }, + event_sender.clone(), )?; - if let Some(event_sender) = event_sender.clone() { - task_handles.append(&mut monitor_file_urls( - &[ - report_config - .no_repro - .clone() - .and_then(|u| u.url.as_file_path()), - report_config - .reports - .clone() - .and_then(|u| u.url.as_file_path()), - report_config - .unique_reports - .clone() - .and_then(|u| u.url.as_file_path()), - ], - event_sender, - )); - } let mut report = ReportTask::new(report_config); let report_task = spawn(async move { report.managed_run().await }); @@ -111,24 +83,9 @@ pub async fn run( task_id: Uuid::new_v4(), ..context.common_config.clone() }, + event_sender.clone(), )?; - if let Some(event_sender) = event_sender { - task_handles.append(&mut monitor_file_urls( - &coverage_config - .readonly_inputs - .iter() - .cloned() - .map(|input| input.url.as_file_path()) - .collect::>(), - event_sender.clone(), - )); - task_handles.append(&mut monitor_file_urls( - &[coverage_config.coverage.url.as_file_path()], - event_sender, - )); - } - let mut coverage = CoverageTask::new(coverage_config); let coverage_task = spawn(async move { coverage.managed_run().await }); @@ -145,6 +102,7 @@ pub async fn run( task_id: Uuid::new_v4(), ..context.common_config.clone() }, + event_sender, )?; let analysis_task = spawn(async move { run_analysis(analysis_config).await }); diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 48d70dc358..a724e74c82 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -15,12 +15,16 @@ use crate::{ use anyhow::Result; use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; +use tokio::sync::mpsc::UnboundedSender; + +use super::common::{SyncCountDirMonitor, UiEvent}; pub fn build_coverage_config( args: &clap::ArgMatches<'_>, local_job: bool, input_queue: Option, common: CommonConfig, + event_sender: Option>, ) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; @@ -35,9 +39,13 @@ pub fn build_coverage_config( )?] } else { get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)? + .into_iter() + .map(|sd| sd.monitor_count(&event_sender)) + .collect::>>()? }; - let coverage = get_synced_dir(COVERAGE_DIR, common.job_id, common.task_id, args)?; + let coverage = get_synced_dir(COVERAGE_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let check_fuzzer_help = args.is_present(CHECK_FUZZER_HELP); let config = Config { @@ -51,12 +59,22 @@ pub fn build_coverage_config( common, check_queue: false, }; + Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let config = build_coverage_config(args, false, None, context.common_config.clone())?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_coverage_config( + args, + false, + None, + context.common_config.clone(), + event_sender, + )?; let mut task = CoverageTask::new(config); task.managed_run().await diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index d3f3a88237..84f724c222 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -4,8 +4,9 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, - REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + SyncCountDirMonitor, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, CRASHES_DIR, + DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, + TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::{ config::CommonConfig, @@ -15,23 +16,34 @@ use crate::{ use anyhow::Result; use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; +use tokio::sync::mpsc::UnboundedSender; + +use super::common::UiEvent; pub fn build_report_config( args: &clap::ArgMatches<'_>, input_queue: Option, common: CommonConfig, + event_sender: Option>, ) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; let target_options = get_cmd_arg(CmdType::Target, args); - let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args).ok(); - let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args).ok(); + let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; + let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; - let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args).ok(); + let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; - let unique_reports = - get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args).ok(); + let unique_reports = get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); @@ -59,12 +71,16 @@ pub fn build_report_config( unique_reports, common, }; + Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let config = build_report_config(args, None, context.common_config.clone())?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_report_config(args, None, context.common_config.clone(), event_sender)?; ReportTask::new(config).managed_run().await } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index 31a88d81c2..d1f3be5168 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -4,8 +4,8 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, - TARGET_WORKERS, + SyncCountDirMonitor, CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, + TARGET_OPTIONS, TARGET_WORKERS, }, tasks::{ config::CommonConfig, @@ -14,11 +14,19 @@ use crate::{ }; use anyhow::Result; use clap::{App, Arg, SubCommand}; +use tokio::sync::mpsc::UnboundedSender; + +use super::common::UiEvent; const DISABLE_EXPECT_CRASH_ON_FAILURE: &str = "disable_expect_crash_on_failure"; -pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> Result { - let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?; +pub fn build_fuzz_config( + args: &clap::ArgMatches<'_>, + common: CommonConfig, + event_sender: Option>, +) -> Result { + let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let inputs = get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)?; let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); @@ -49,9 +57,12 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>, common: CommonConfig) -> R Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let config = build_fuzz_config(args, context.common_config.clone())?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_fuzz_config(args, context.common_config.clone(), event_sender)?; LibFuzzerFuzzTask::new(config)?.run().await } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs index cf352f0b09..ebe1451027 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs @@ -4,8 +4,9 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, - get_synced_dirs, CmdType, ANALYSIS_INPUTS, ANALYSIS_UNIQUE_INPUTS, CHECK_FUZZER_HELP, - INPUTS_DIR, PRESERVE_EXISTING_OUTPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, + get_synced_dirs, CmdType, SyncCountDirMonitor, ANALYSIS_INPUTS, ANALYSIS_UNIQUE_INPUTS, + CHECK_FUZZER_HELP, INPUTS_DIR, PRESERVE_EXISTING_OUTPUTS, TARGET_ENV, TARGET_EXE, + TARGET_OPTIONS, }, tasks::{ config::CommonConfig, @@ -15,11 +16,15 @@ use crate::{ use anyhow::Result; use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; +use tokio::sync::mpsc::UnboundedSender; + +use super::common::UiEvent; pub fn build_merge_config( args: &clap::ArgMatches<'_>, input_queue: Option, common: CommonConfig, + event_sender: Option>, ) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; @@ -27,7 +32,8 @@ pub fn build_merge_config( let check_fuzzer_help = args.is_present(CHECK_FUZZER_HELP); let inputs = get_synced_dirs(ANALYSIS_INPUTS, common.job_id, common.task_id, args)?; let unique_inputs = - get_synced_dir(ANALYSIS_UNIQUE_INPUTS, common.job_id, common.task_id, args)?; + get_synced_dir(ANALYSIS_UNIQUE_INPUTS, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let preserve_existing_outputs = value_t!(args, PRESERVE_EXISTING_OUTPUTS, bool)?; let config = Config { @@ -41,12 +47,16 @@ pub fn build_merge_config( unique_inputs, preserve_existing_outputs, }; + Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let config = build_merge_config(args, None, context.common_config.clone())?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_merge_config(args, None, context.common_config.clone(), event_sender)?; spawn(std::sync::Arc::new(config)).await } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs index 823c27bdea..e33e70f1ce 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR, + UiEvent, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR, REGRESSION_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, @@ -15,6 +15,7 @@ use crate::{ }; use anyhow::Result; use clap::{App, Arg, SubCommand}; +use tokio::sync::mpsc::UnboundedSender; const REPORT_NAMES: &str = "report_names"; @@ -64,8 +65,11 @@ pub fn build_regression_config( Ok(config) } -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender)?; let config = build_regression_config(args, context.common_config.clone())?; LibFuzzerRegressionTask::new(config).run().await } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs index 323110ea5c..2e46cce097 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs @@ -11,9 +11,15 @@ use crate::{ use anyhow::Result; use clap::{App, Arg, SubCommand}; use std::path::PathBuf; +use tokio::sync::mpsc::UnboundedSender; -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; +use super::common::UiEvent; + +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender)?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_env = get_cmd_env(CmdType::Target, args)?; diff --git a/src/agent/onefuzz-agent/src/local/radamsa.rs b/src/agent/onefuzz-agent/src/local/radamsa.rs index 3e69d2aba9..2447abe87c 100644 --- a/src/agent/onefuzz-agent/src/local/radamsa.rs +++ b/src/agent/onefuzz-agent/src/local/radamsa.rs @@ -13,13 +13,18 @@ use anyhow::Result; use clap::{App, SubCommand}; use onefuzz::utils::try_wait_all_join_handles; use std::collections::HashSet; +use tokio::sync::mpsc::UnboundedSender; use tokio::task::spawn; - use uuid::Uuid; -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, true)?; - let fuzz_config = build_fuzz_config(args, context.common_config.clone())?; +use super::common::UiEvent; + +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, true, event_sender.clone())?; + let fuzz_config = build_fuzz_config(args, context.common_config.clone(), event_sender.clone())?; let crash_dir = fuzz_config .crashes .url @@ -37,6 +42,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { task_id: Uuid::new_v4(), ..context.common_config.clone() }, + event_sender, )?; let report_task = spawn(async move { ReportTask::new(report_config).managed_run().await }); diff --git a/src/agent/onefuzz-agent/src/local/test_input.rs b/src/agent/onefuzz-agent/src/local/test_input.rs index 95973dcbb5..9039ac33e1 100644 --- a/src/agent/onefuzz-agent/src/local/test_input.rs +++ b/src/agent/onefuzz-agent/src/local/test_input.rs @@ -11,9 +11,15 @@ use crate::{ use anyhow::Result; use clap::{App, Arg, SubCommand}; use std::path::PathBuf; +use tokio::sync::mpsc::UnboundedSender; -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let context = build_local_context(args, false)?; +use super::common::UiEvent; + +pub async fn run( + args: &clap::ArgMatches<'_>, + event_sender: Option>, +) -> Result<()> { + let context = build_local_context(args, false, event_sender)?; let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; let target_env = get_cmd_env(CmdType::Target, args)?; diff --git a/src/agent/onefuzz-agent/src/local/tui.rs b/src/agent/onefuzz-agent/src/local/tui.rs index b188ee534e..88db0bf655 100644 --- a/src/agent/onefuzz-agent/src/local/tui.rs +++ b/src/agent/onefuzz-agent/src/local/tui.rs @@ -11,6 +11,7 @@ use crossterm::{ use futures::{StreamExt, TryStreamExt}; use log::Level; use onefuzz::utils::try_wait_all_join_handles; +use onefuzz_telemetry::{self, EventData}; use std::{ collections::HashMap, io::{self, Stdout, Write}, @@ -19,8 +20,11 @@ use std::{ time::Duration, }; use tokio::{ - sync::mpsc::{self, UnboundedReceiver}, - time, + sync::{ + broadcast::{self, RecvError}, + mpsc::{self, UnboundedSender}, + }, + time::delay_for, }; use tui::{ backend::CrosstermBackend, @@ -28,12 +32,14 @@ use tui::{ style::{Color, Modifier, Style}, text::{Span, Spans}, widgets::{Block, Borders}, - widgets::{List, ListItem, ListState}, + widgets::{Gauge, List, ListItem, ListState}, Terminal, }; use arraydeque::{ArrayDeque, Wrapping}; +use super::common::wait_for_dir; + #[derive(Debug, thiserror::Error)] enum UiLoopError { #[error("program exiting")] @@ -58,6 +64,13 @@ impl From for UiLoopError { const LOGS_BUFFER_SIZE: usize = 100; const TICK_RATE: Duration = Duration::from_millis(250); +#[derive(Debug, Default)] +struct CoverageData { + covered: Option, + features: Option, + rate: Option, +} + /// Event driving the refresh of the UI #[derive(Debug)] enum TerminalEvent { @@ -65,15 +78,19 @@ enum TerminalEvent { Tick, FileCount { dir: PathBuf, count: usize }, Quit, + MonitorDir(PathBuf), + Coverage(CoverageData), } struct UiLoopState { pub logs: ArrayDeque<[(Level, String); LOGS_BUFFER_SIZE], Wrapping>, pub file_count: HashMap, pub file_count_state: ListState, - pub file_monitors: Vec>>, + pub file_monitors: HashMap>>, pub log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>, pub terminal: Terminal>, + pub coverage: CoverageData, + pub cancellation_tx: broadcast::Sender<()>, } impl UiLoopState { @@ -81,6 +98,7 @@ impl UiLoopState { terminal: Terminal>, log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>, ) -> Self { + let (cancellation_tx, _) = broadcast::channel(1); Self { log_event_receiver, logs: Default::default(), @@ -88,6 +106,8 @@ impl UiLoopState { file_count_state: Default::default(), file_monitors: Default::default(), terminal, + coverage: Default::default(), + cancellation_tx, } } } @@ -130,40 +150,97 @@ impl TerminalUi { .init(); let tick_event_tx_clone = self.ui_event_tx.clone(); - let tick_event_handle = - tokio::spawn(async { Self::ticking(tick_event_tx_clone).await.context("ticking") }); + let tick_event_handle = tokio::spawn(Self::ticking( + tick_event_tx_clone, + initial_state.cancellation_tx.subscribe(), + )); let keyboard_ui_event_tx = self.ui_event_tx.clone(); - let _keyboard_event_handle = Self::read_keyboard_events(keyboard_ui_event_tx); + let _keyboard_event_handle = Self::read_keyboard_events( + keyboard_ui_event_tx, + initial_state.cancellation_tx.subscribe(), + ); let task_event_receiver = self.task_event_receiver; let ui_event_tx = self.ui_event_tx.clone(); - let external_event_handle = - tokio::spawn(Self::read_commands(ui_event_tx, task_event_receiver)); - let ui_loop = tokio::spawn(Self::ui_loop(initial_state, self.ui_event_rx)); + let external_event_handle = tokio::spawn(Self::read_commands( + ui_event_tx, + task_event_receiver, + initial_state.cancellation_tx.subscribe(), + )); + + let mut task_handles = vec![tick_event_handle, external_event_handle]; + + let ui_event_tx = self.ui_event_tx.clone(); + let telemetry = tokio::spawn(Self::listen_telemetry_event( + ui_event_tx, + initial_state.cancellation_tx.subscribe(), + )); + + task_handles.push(telemetry); + + let ui_loop = tokio::spawn(Self::ui_loop( + initial_state, + self.ui_event_rx, + self.ui_event_tx.clone(), + )); - let mut task_handles = vec![tick_event_handle, ui_loop, external_event_handle]; + task_handles.push(ui_loop); if let Some(timeout) = timeout { let ui_event_tx = self.ui_event_tx.clone(); - let timeout_task = tokio::spawn(async move { - time::delay_for(timeout).await; + tokio::spawn(async move { + tokio::time::delay_for(timeout).await; let _ = ui_event_tx.send(TerminalEvent::Quit); - Ok(()) }); - task_handles.push(timeout_task); } - try_wait_all_join_handles(task_handles) - .await - .context("ui_loop")?; + try_wait_all_join_handles(task_handles).await?; + Ok(()) + } + + async fn listen_telemetry_event( + ui_event_tx: UnboundedSender, + mut cancellation_rx: broadcast::Receiver<()>, + ) -> Result<()> { + let mut rx = onefuzz_telemetry::subscribe_to_events(); + + while cancellation_rx.try_recv() == Err(broadcast::TryRecvError::Empty) { + match rx.recv().await { + Ok((event, data)) => { + if let onefuzz_telemetry::Event::coverage_data = event { + let (covered, features, rate) = + data.iter() + .cloned() + .fold((None, None, None), |(c, f, r), d| match d { + EventData::Covered(value) => (Some(value), f, r), + EventData::Features(value) => (c, Some(value), r), + EventData::Rate(value) => (c, f, Some(value)), + _ => (c, f, r), + }); + + let _ = ui_event_tx.send(TerminalEvent::Coverage(CoverageData { + covered, + features, + rate, + })); + } + } + + Err(RecvError::Lagged(_)) => continue, + Err(RecvError::Closed) => break, + } + } Ok(()) } - async fn ticking(ui_event_tx: mpsc::UnboundedSender) -> Result<()> { + async fn ticking( + ui_event_tx: mpsc::UnboundedSender, + mut cancellation_rx: broadcast::Receiver<()>, + ) -> Result<()> { let mut interval = tokio::time::interval(TICK_RATE); - loop { + while Err(broadcast::TryRecvError::Empty) == cancellation_rx.try_recv() { interval.tick().await; if let Err(_err) = ui_event_tx.send(TerminalEvent::Tick) { break; @@ -174,34 +251,42 @@ impl TerminalUi { fn read_keyboard_events( ui_event_tx: mpsc::UnboundedSender, + mut cancellation_rx: broadcast::Receiver<()>, ) -> JoinHandle> { - thread::spawn(move || loop { - if event::poll(Duration::from_secs(1))? { - let event = event::read()?; - if let Err(_err) = ui_event_tx.send(TerminalEvent::Input(event)) { - return Ok(()); + thread::spawn(move || { + while Err(broadcast::TryRecvError::Empty) == cancellation_rx.try_recv() { + if event::poll(Duration::from_secs(1))? { + let event = event::read()?; + if let Err(_err) = ui_event_tx.send(TerminalEvent::Input(event)) { + return Ok(()); + } } } + Ok(()) }) } async fn read_commands( ui_event_tx: mpsc::UnboundedSender, mut external_event_rx: mpsc::UnboundedReceiver, + mut cancellation_rx: broadcast::Receiver<()>, ) -> Result<()> { - while let Some(UiEvent::FileCount { dir, count }) = external_event_rx.recv().await { - if ui_event_tx - .send(TerminalEvent::FileCount { dir, count }) - .is_err() - { - break; + while Err(broadcast::TryRecvError::Empty) == cancellation_rx.try_recv() { + match external_event_rx.try_recv() { + Ok(UiEvent::MonitorDir(dir)) => { + if ui_event_tx.send(TerminalEvent::MonitorDir(dir)).is_err() { + break; + } + } + Err(mpsc::error::TryRecvError::Empty) => delay_for(Duration::from_secs(1)).await, + Err(mpsc::error::TryRecvError::Closed) => break, } } Ok(()) } fn take_available_logs( - receiver: &mut UnboundedReceiver, + receiver: &mut mpsc::UnboundedReceiver, size: usize, buffer: &mut ArrayDeque<[T; LOGS_BUFFER_SIZE], Wrapping>, ) { @@ -221,6 +306,9 @@ impl TerminalUi { let file_count = ui_state.file_count; let mut log_event_receiver = ui_state.log_event_receiver; let mut terminal = ui_state.terminal; + let rate = ui_state.coverage.rate.unwrap_or(0.0); + let features = ui_state.coverage.features.unwrap_or(0); + let covered = ui_state.coverage.covered.unwrap_or(0); Self::take_available_logs(&mut log_event_receiver, 10, &mut logs); terminal.draw(|f| { @@ -229,6 +317,49 @@ impl TerminalUi { .constraints([Constraint::Percentage(25), Constraint::Percentage(75)].as_ref()) .split(f.size()); + let log_area = chunks[1]; + let top_area = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref()) + .split(chunks[0]); + + let file_count_area = top_area[0]; + let coverage_area = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Percentage(25), Constraint::Percentage(75)].as_ref()) + .margin(1) + .split(top_area[1]); + + let label = format!("{:.2}%", rate * 100.0); + + let coverage_block = Block::default().borders(Borders::ALL).title("Coverage:"); + + f.render_widget(coverage_block, top_area[1]); + + let gauge = Gauge::default() + .gauge_style( + Style::default() + .fg(Color::Magenta) + .bg(Color::Black) + .add_modifier(Modifier::ITALIC | Modifier::BOLD), + ) + .label(label) + .ratio(rate); + f.render_widget(gauge, coverage_area[0]); + + let coverage_info = List::new([ + ListItem::new(Spans::from(vec![ + Span::raw("features: "), + Span::raw(format!("{}", features)), + ])), + ListItem::new(Spans::from(vec![ + Span::raw("covered: "), + Span::raw(format!("{}", covered)), + ])), + ]); + + f.render_widget(coverage_info, coverage_area[1]); + let mut sorted_file_count = file_count.iter().collect::>(); sorted_file_count.sort_by(|(p1, _), (p2, _)| p1.cmp(p2)); @@ -253,7 +384,7 @@ impl TerminalUi { .highlight_style(Style::default().add_modifier(Modifier::BOLD)) .start_corner(Corner::TopLeft); - f.render_stateful_widget(log_list, chunks[0], &mut file_count_state); + f.render_stateful_widget(log_list, file_count_area, &mut file_count_state); let log_items = logs .iter() @@ -278,7 +409,7 @@ impl TerminalUi { .block(Block::default().borders(Borders::ALL).title("Logs")) .start_corner(Corner::BottomLeft); - f.render_widget(log_list, chunks[1]); + f.render_widget(log_list, log_area); })?; Ok(UiLoopState { logs, @@ -331,11 +462,24 @@ impl TerminalUi { }) } - async fn on_quit(ui_state: UiLoopState) -> Result { + async fn on_quit( + ui_state: UiLoopState, + cancellation_tx: broadcast::Sender<()>, + ) -> Result { + let _ = cancellation_tx.send(()); + let file_monitors = ui_state.file_monitors.into_iter().map(|(_, v)| v).collect(); + tokio::time::timeout( + Duration::from_secs(10), + try_wait_all_join_handles(file_monitors), + ) + .await + .context("failed to close file monitoring tasks")? + .context("file monitoring task terminated with error")?; let mut terminal = ui_state.terminal; disable_raw_mode().map_err(|e| anyhow!("{:?}", e))?; execute!(terminal.backend_mut(), LeaveAlternateScreen).map_err(|e| anyhow!("{:?}", e))?; terminal.show_cursor()?; + Err(UiLoopError::Exit) } @@ -352,17 +496,38 @@ impl TerminalUi { }) } + async fn on_monitor_dir( + ui_state: UiLoopState, + path: PathBuf, + ui_event_tx: mpsc::UnboundedSender, + cancellation_rx: broadcast::Receiver<()>, + ) -> Result { + let mut file_monitors = ui_state.file_monitors; + + file_monitors.entry(path).or_insert_with_key(|path| { + Self::spawn_file_count_monitor(path.clone(), ui_event_tx, cancellation_rx) + }); + + Ok(UiLoopState { + file_monitors, + ..ui_state + }) + } + async fn ui_loop( initial_state: UiLoopState, ui_event_rx: mpsc::UnboundedReceiver, + ui_event_tx: mpsc::UnboundedSender, ) -> Result<()> { let loop_result = ui_event_rx .map(Ok) .try_fold(initial_state, |ui_state, event| async { + let ui_event_tx = ui_event_tx.clone(); + let cancellation_tx = ui_state.cancellation_tx.clone(); match event { TerminalEvent::Tick => Self::refresh_ui(ui_state).await, TerminalEvent::Input(Event::Key(k)) => match k.code { - KeyCode::Char('q') => Self::on_quit(ui_state).await, + KeyCode::Char('q') => Self::on_quit(ui_state, cancellation_tx).await, KeyCode::Down => Self::on_key_down(ui_state).await, KeyCode::Up => Self::on_key_up(ui_state).await, _ => Ok(ui_state), @@ -370,7 +535,20 @@ impl TerminalUi { TerminalEvent::FileCount { dir, count } => { Self::on_file_count(ui_state, dir, count).await } - TerminalEvent::Quit => Self::on_quit(ui_state).await, + TerminalEvent::Quit => Self::on_quit(ui_state, cancellation_tx).await, + TerminalEvent::MonitorDir(path) => { + Self::on_monitor_dir( + ui_state, + path, + ui_event_tx, + cancellation_tx.subscribe(), + ) + .await + } + TerminalEvent::Coverage(coverage) => Ok(UiLoopState { + coverage, + ..ui_state + }), _ => Ok(ui_state), } }) @@ -381,4 +559,36 @@ impl TerminalUi { Err(UiLoopError::Anyhow(e)) => Err(e), } } + + fn spawn_file_count_monitor( + dir: PathBuf, + sender: mpsc::UnboundedSender, + mut cancellation_rx: broadcast::Receiver<()>, + ) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + wait_for_dir(&dir).await?; + while cancellation_rx.try_recv() == Err(broadcast::TryRecvError::Empty) { + let mut rd = tokio::fs::read_dir(&dir).await?; + let mut count: usize = 0; + + while let Some(Ok(entry)) = rd.next().await { + if entry.path().is_file() { + count += 1; + } + } + + if sender + .send(TerminalEvent::FileCount { + dir: dir.clone(), + count, + }) + .is_err() + { + break; + } + delay_for(Duration::from_secs(5)).await; + } + Ok(()) + }) + } } diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index 27f6aa8c93..0a06af67a9 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -250,7 +250,11 @@ impl LibFuzzerFuzzTask { for file in &files { if let Some(filename) = file.file_name() { let dest = self.config.crashes.path.join(filename); - tokio::fs::rename(file, dest).await?; + if let Err(e) = tokio::fs::rename(file.clone(), dest.clone()).await { + if !dest.exists() { + bail!(e) + } + } } } diff --git a/src/agent/onefuzz-telemetry/Cargo.toml b/src/agent/onefuzz-telemetry/Cargo.toml index 60e0827ca0..3b607f42d2 100644 --- a/src/agent/onefuzz-telemetry/Cargo.toml +++ b/src/agent/onefuzz-telemetry/Cargo.toml @@ -17,7 +17,7 @@ uuid = { version = "0.8", features = ["serde", "v4"] } serde = { version = "1.0", features = ["derive"] } z3-sys = { version = "0.6", optional = true} iced-x86 = { version = "1.1", optional = true} +tokio = { version = "0.2" } +lazy_static = "1.4" -[dev-dependencies] -tokio = { version = "0.2" } diff --git a/src/agent/onefuzz-telemetry/src/lib.rs b/src/agent/onefuzz-telemetry/src/lib.rs index 72ef089baa..d6956168fd 100644 --- a/src/agent/onefuzz-telemetry/src/lib.rs +++ b/src/agent/onefuzz-telemetry/src/lib.rs @@ -11,6 +11,9 @@ use uuid::Uuid; use z3_sys::ErrorCode as Z3ErrorCode; pub use appinsights::telemetry::SeverityLevel::{Critical, Error, Information, Verbose, Warning}; +use tokio::sync::broadcast::{self, Receiver}; +#[macro_use] +extern crate lazy_static; #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(transparent)] @@ -338,6 +341,8 @@ mod global { RwLock, }; + use tokio::sync::broadcast::Sender; + use super::*; #[derive(Default)] @@ -350,6 +355,14 @@ mod global { instance: None, microsoft: None, }; + + lazy_static! { + pub static ref EVENT_SOURCE: Sender<(Event, Vec)> = { + let (telemetry_event_source, _) = broadcast::channel::<_>(100); + telemetry_event_source + }; + } + const UNSET: usize = 0; const SETTING: usize = 1; const SET: usize = 2; @@ -501,6 +514,17 @@ pub fn format_events(events: &[EventData]) -> String { .join(" ") } +fn try_broadcast_event(event: &Event, properties: &[EventData]) -> bool { + // we ignore any send error here because they indicate that + // there are no receivers on the other end + let (event, properties) = (event.clone(), properties.to_vec()); + global::EVENT_SOURCE.send((event, properties)).is_ok() +} + +pub fn subscribe_to_events() -> Receiver<(Event, Vec)> { + global::EVENT_SOURCE.subscribe() +} + pub fn track_event(event: &Event, properties: &[EventData]) { use appinsights::telemetry::Telemetry; @@ -526,6 +550,7 @@ pub fn track_event(event: &Event, properties: &[EventData]) { } client.track(evt); } + try_broadcast_event(event, properties); } pub fn to_log_level(level: &appinsights::telemetry::SeverityLevel) -> log::Level { From 16c70d175f3a0d5b59be2777c25a171ed0dd168a Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 6 Apr 2021 15:05:46 -0700 Subject: [PATCH 2/7] formatting --- src/agent/onefuzz-agent/src/local/common.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index e8b41c487a..ddd7e1b8d1 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -208,7 +208,6 @@ pub fn get_synced_dir( }) } - // NOTE: generate_task_id is intended to change the default behavior for local // fuzzing tasks from generating random task id to using UUID::nil(). This // enables making the one-shot crash report generation, which isn't really a task, From 0c2919e6b8d5d702335b25f604e5ef7f319d3224 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 6 Apr 2021 16:37:19 -0700 Subject: [PATCH 3/7] monitoring more files --- .../src/local/generic_generator.rs | 6 ++++-- .../src/local/libfuzzer_coverage.rs | 2 +- .../onefuzz-agent/src/local/libfuzzer_fuzz.rs | 2 +- .../onefuzz-agent/src/local/libfuzzer_merge.rs | 4 +++- .../src/local/libfuzzer_regression.rs | 17 +++++++++-------- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index 6462b744b2..080480ac5e 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -34,7 +34,9 @@ pub fn build_fuzz_config( let generator_exe = get_cmd_exe(CmdType::Generator, args)?; let generator_options = get_cmd_arg(CmdType::Generator, args); let generator_env = get_cmd_env(CmdType::Generator, args)?; - let readonly_inputs = get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)?; + let readonly_inputs = get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)?.into_iter() + .map(|sd| sd.monitor_count(&event_sender)) + .collect::>>()?; let rename_output = args.is_present(RENAME_OUTPUT); let check_asan_log = args.is_present(CHECK_ASAN_LOG); @@ -42,7 +44,7 @@ pub fn build_fuzz_config( let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; let target_timeout = Some(value_t!(args, TARGET_TIMEOUT, u64)?); - let tools = get_synced_dir(TOOLS_DIR, common.job_id, common.task_id, args).ok(); + let tools = get_synced_dir(TOOLS_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; let ensemble_sync_delay = None; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index a724e74c82..0625c7c601 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -36,7 +36,7 @@ pub fn build_coverage_config( common.job_id, common.task_id, args, - )?] + )?.monitor_count(&event_sender)?] } else { get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)? .into_iter() diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index d1f3be5168..c93e14fd32 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -27,7 +27,7 @@ pub fn build_fuzz_config( ) -> Result { let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? .monitor_count(&event_sender)?; - let inputs = get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)?; + let inputs = get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)?.monitor_count(&event_sender)?; let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs index ebe1451027..678ed71466 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs @@ -30,7 +30,9 @@ pub fn build_merge_config( let target_env = get_cmd_env(CmdType::Target, args)?; let target_options = get_cmd_arg(CmdType::Target, args); let check_fuzzer_help = args.is_present(CHECK_FUZZER_HELP); - let inputs = get_synced_dirs(ANALYSIS_INPUTS, common.job_id, common.task_id, args)?; + let inputs = get_synced_dirs(ANALYSIS_INPUTS, common.job_id, common.task_id, args)?.into_iter() + .map(|sd| sd.monitor_count(&event_sender)) + .collect::>>()?; let unique_inputs = get_synced_dir(ANALYSIS_UNIQUE_INPUTS, common.job_id, common.task_id, args)? .monitor_count(&event_sender)?; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs index e33e70f1ce..bf77230e36 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - UiEvent, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR, + UiEvent, SyncCountDirMonitor, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR, REGRESSION_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, @@ -22,20 +22,21 @@ const REPORT_NAMES: &str = "report_names"; pub fn build_regression_config( args: &clap::ArgMatches<'_>, common: CommonConfig, + event_sender: Option>, ) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; let target_options = get_cmd_arg(CmdType::Target, args); let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); - let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?; + let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?.monitor_count(&event_sender)?; let regression_reports = - get_synced_dir(REGRESSION_REPORTS_DIR, common.job_id, common.task_id, args)?; + get_synced_dir(REGRESSION_REPORTS_DIR, common.job_id, common.task_id, args)?.monitor_count(&event_sender)?; let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; - let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args).ok(); - let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args).ok(); + let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; + let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; let unique_reports = - get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args).ok(); + get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; let report_list = if args.is_present(REPORT_NAMES) { Some(values_t!(args, REPORT_NAMES, String)?) @@ -69,8 +70,8 @@ pub async fn run( args: &clap::ArgMatches<'_>, event_sender: Option>, ) -> Result<()> { - let context = build_local_context(args, true, event_sender)?; - let config = build_regression_config(args, context.common_config.clone())?; + let context = build_local_context(args, true, event_sender.clone())?; + let config = build_regression_config(args, context.common_config.clone(), event_sender)?; LibFuzzerRegressionTask::new(config).run().await } From f487afb91bc1ea82b89bde94a78ee1a0c393e71c Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 6 Apr 2021 16:42:09 -0700 Subject: [PATCH 4/7] format --- .../src/local/generic_generator.rs | 11 +++++--- .../onefuzz-agent/src/local/libfuzzer.rs | 3 ++- .../src/local/libfuzzer_coverage.rs | 10 +++----- .../onefuzz-agent/src/local/libfuzzer_fuzz.rs | 3 ++- .../src/local/libfuzzer_merge.rs | 7 +++--- .../src/local/libfuzzer_regression.rs | 25 ++++++++++++------- 6 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index 080480ac5e..b5589d2838 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -34,9 +34,10 @@ pub fn build_fuzz_config( let generator_exe = get_cmd_exe(CmdType::Generator, args)?; let generator_options = get_cmd_arg(CmdType::Generator, args); let generator_env = get_cmd_env(CmdType::Generator, args)?; - let readonly_inputs = get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)?.into_iter() - .map(|sd| sd.monitor_count(&event_sender)) - .collect::>>()?; + let readonly_inputs = get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)? + .into_iter() + .map(|sd| sd.monitor_count(&event_sender)) + .collect::>>()?; let rename_output = args.is_present(RENAME_OUTPUT); let check_asan_log = args.is_present(CHECK_ASAN_LOG); @@ -44,7 +45,9 @@ pub fn build_fuzz_config( let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; let target_timeout = Some(value_t!(args, TARGET_TIMEOUT, u64)?); - let tools = get_synced_dir(TOOLS_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; + let tools = get_synced_dir(TOOLS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; let ensemble_sync_delay = None; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index cb519fb235..2f1589a73f 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -102,7 +102,7 @@ pub async fn run( task_id: Uuid::new_v4(), ..context.common_config.clone() }, - event_sender, + event_sender.clone(), )?; let analysis_task = spawn(async move { run_analysis(analysis_config).await }); @@ -117,6 +117,7 @@ pub async fn run( task_id: Uuid::new_v4(), ..context.common_config.clone() }, + event_sender, )?; let regression = LibFuzzerRegressionTask::new(regression_config); let regression_task = spawn(async move { regression.run().await }); diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 0625c7c601..a7ab89ce78 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -31,12 +31,10 @@ pub fn build_coverage_config( let target_options = get_cmd_arg(CmdType::Target, args); let readonly_inputs = if local_job { - vec![get_synced_dir( - INPUTS_DIR, - common.job_id, - common.task_id, - args, - )?.monitor_count(&event_sender)?] + vec![ + get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?, + ] } else { get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)? .into_iter() diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index c93e14fd32..2ca0dbcabb 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -27,7 +27,8 @@ pub fn build_fuzz_config( ) -> Result { let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? .monitor_count(&event_sender)?; - let inputs = get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)?.monitor_count(&event_sender)?; + let inputs = get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs index 678ed71466..a72dc78b1b 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs @@ -30,9 +30,10 @@ pub fn build_merge_config( let target_env = get_cmd_env(CmdType::Target, args)?; let target_options = get_cmd_arg(CmdType::Target, args); let check_fuzzer_help = args.is_present(CHECK_FUZZER_HELP); - let inputs = get_synced_dirs(ANALYSIS_INPUTS, common.job_id, common.task_id, args)?.into_iter() - .map(|sd| sd.monitor_count(&event_sender)) - .collect::>>()?; + let inputs = get_synced_dirs(ANALYSIS_INPUTS, common.job_id, common.task_id, args)? + .into_iter() + .map(|sd| sd.monitor_count(&event_sender)) + .collect::>>()?; let unique_inputs = get_synced_dir(ANALYSIS_UNIQUE_INPUTS, common.job_id, common.task_id, args)? .monitor_count(&event_sender)?; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs index bf77230e36..7c916ace6f 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_regression.rs @@ -4,9 +4,9 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - UiEvent, SyncCountDirMonitor, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, NO_REPRO_DIR, - REGRESSION_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, - TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + SyncCountDirMonitor, UiEvent, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, COVERAGE_DIR, + CRASHES_DIR, NO_REPRO_DIR, REGRESSION_REPORTS_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, + TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::{ config::CommonConfig, @@ -28,15 +28,22 @@ pub fn build_regression_config( let target_env = get_cmd_env(CmdType::Target, args)?; let target_options = get_cmd_arg(CmdType::Target, args); let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); - let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?.monitor_count(&event_sender)?; + let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let regression_reports = - get_synced_dir(REGRESSION_REPORTS_DIR, common.job_id, common.task_id, args)?.monitor_count(&event_sender)?; + get_synced_dir(REGRESSION_REPORTS_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; - let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; - let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; - let unique_reports = - get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args).ok().monitor_count(&event_sender)?; + let reports = get_synced_dir(REPORTS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; + let no_repro = get_synced_dir(NO_REPRO_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; + let unique_reports = get_synced_dir(UNIQUE_REPORTS_DIR, common.job_id, common.task_id, args) + .ok() + .monitor_count(&event_sender)?; let report_list = if args.is_present(REPORT_NAMES) { Some(values_t!(args, REPORT_NAMES, String)?) From f4c59d73e80ff9e3830c37137307e9733ad84b5b Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Wed, 7 Apr 2021 18:13:44 -0700 Subject: [PATCH 5/7] - split the top area horisontally - swithed to using paragraph to display file count and stats - change color of coverage bar to white --- src/agent/onefuzz-agent/src/local/tui.rs | 310 ++++++++++++++--------- 1 file changed, 195 insertions(+), 115 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/tui.rs b/src/agent/onefuzz-agent/src/local/tui.rs index 88db0bf655..4aefb0c87e 100644 --- a/src/agent/onefuzz-agent/src/local/tui.rs +++ b/src/agent/onefuzz-agent/src/local/tui.rs @@ -15,6 +15,8 @@ use onefuzz_telemetry::{self, EventData}; use std::{ collections::HashMap, io::{self, Stdout, Write}, + iter::once, + mem::{discriminant, Discriminant}, path::PathBuf, thread::{self, JoinHandle}, time::Duration, @@ -28,11 +30,11 @@ use tokio::{ }; use tui::{ backend::CrosstermBackend, - layout::{Constraint, Corner, Direction, Layout}, + layout::{Alignment, Constraint, Corner, Direction, Layout}, style::{Color, Modifier, Style}, text::{Span, Spans}, widgets::{Block, Borders}, - widgets::{Gauge, List, ListItem, ListState}, + widgets::{Gauge, List, ListItem, ListState, Paragraph, Wrap}, Terminal, }; @@ -79,7 +81,7 @@ enum TerminalEvent { FileCount { dir: PathBuf, count: usize }, Quit, MonitorDir(PathBuf), - Coverage(CoverageData), + Telemetry(Vec), } struct UiLoopState { @@ -89,8 +91,8 @@ struct UiLoopState { pub file_monitors: HashMap>>, pub log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>, pub terminal: Terminal>, - pub coverage: CoverageData, pub cancellation_tx: broadcast::Sender<()>, + pub events: HashMap, EventData>, } impl UiLoopState { @@ -99,6 +101,7 @@ impl UiLoopState { log_event_receiver: mpsc::UnboundedReceiver<(Level, String)>, ) -> Self { let (cancellation_tx, _) = broadcast::channel(1); + let events = HashMap::new(); Self { log_event_receiver, logs: Default::default(), @@ -106,8 +109,8 @@ impl UiLoopState { file_count_state: Default::default(), file_monitors: Default::default(), terminal, - coverage: Default::default(), cancellation_tx, + events, } } } @@ -200,6 +203,30 @@ impl TerminalUi { Ok(()) } + fn filter_event(event: &EventData) -> bool { + !matches!( + event, + EventData::WorkerId(_) + | EventData::InstanceId(_) + | EventData::JobId(_) + | EventData::TaskId(_) + | EventData::ScalesetId(_) + | EventData::MachineId(_) + | EventData::Version(_) + | EventData::CommandLine(_) + | EventData::Type(_) + | EventData::Mode(_) + | EventData::Path(_) + | EventData::Count(_) + | EventData::ExecsSecond(_) + | EventData::Pid(_) + | EventData::RunId(_) + | EventData::Name(_) + | EventData::ToolName(_) + | EventData::ProcessStatus(_) + ) + } + async fn listen_telemetry_event( ui_event_tx: UnboundedSender, mut cancellation_rx: broadcast::Receiver<()>, @@ -208,24 +235,12 @@ impl TerminalUi { while cancellation_rx.try_recv() == Err(broadcast::TryRecvError::Empty) { match rx.recv().await { - Ok((event, data)) => { - if let onefuzz_telemetry::Event::coverage_data = event { - let (covered, features, rate) = - data.iter() - .cloned() - .fold((None, None, None), |(c, f, r), d| match d { - EventData::Covered(value) => (Some(value), f, r), - EventData::Features(value) => (c, Some(value), r), - EventData::Rate(value) => (c, f, Some(value)), - _ => (c, f, r), - }); - - let _ = ui_event_tx.send(TerminalEvent::Coverage(CoverageData { - covered, - features, - rate, - })); - } + Ok((_event, data)) => { + let data = data + .into_iter() + .filter(Self::filter_event) + .collect::>(); + let _ = ui_event_tx.send(TerminalEvent::Telemetry(data)); } Err(RecvError::Lagged(_)) => continue, @@ -300,15 +315,141 @@ impl TerminalUi { } } + fn create_coverage_gauge<'a>(rate: f64) -> Gauge<'a> { + let label = format!("coverage {:.2}%", rate * 100.0); + Gauge::default() + .gauge_style( + Style::default() + .fg(Color::White) + .bg(Color::Black) + .add_modifier(Modifier::ITALIC | Modifier::BOLD), + ) + .label(label) + .ratio(rate) + } + + fn create_stats_paragraph( + events: &HashMap, EventData>, + ) -> Paragraph<'_> { + let mut event_values = events.values().map(|v| v.as_values()).collect::>(); + + event_values.sort_by(|(a, _), (b, _)| a.cmp(b)); + + let mut stats_spans = once(Span::styled( + "Stats: ", + Style::default().add_modifier(Modifier::BOLD), + )) + .chain( + event_values + .into_iter() + .map(|(name, value)| { + vec![ + Span::raw(name), + Span::raw(" "), + Span::styled(value, Style::default().add_modifier(Modifier::BOLD)), + Span::raw(", "), + ] + }) + .flatten(), + ) + .collect::>(); + + if stats_spans.len() > 1 { + // removing the last "," + stats_spans.pop(); + } + + Paragraph::new(Spans::from(stats_spans)) + .style(Style::default()) + .alignment(Alignment::Left) + .wrap(Wrap { trim: true }) + } + + fn create_file_count_paragraph(file_count: &HashMap) -> Paragraph<'_> { + let mut sorted_file_count = file_count.iter().collect::>(); + + sorted_file_count.sort_by(|(p1, _), (p2, _)| p1.cmp(p2)); + + let mut files_spans = once(Span::styled( + "Files: ", + Style::default().add_modifier(Modifier::BOLD), + )) + .chain( + sorted_file_count + .iter() + .map(|(path, count)| { + vec![ + Span::raw( + path.file_name() + .map(|f| f.to_string_lossy()) + .unwrap_or_default(), + ), + Span::raw(" "), + Span::styled( + format!("{}", count), + Style::default().add_modifier(Modifier::BOLD), + ), + Span::raw(", "), + ] + }) + .flatten(), + ) + .collect::>(); + + if files_spans.len() > 1 { + files_spans.pop(); + } // removing the last "," + + Paragraph::new(Spans::from(files_spans)) + .style(Style::default()) + .alignment(Alignment::Left) + .wrap(Wrap { trim: true }) + } + + fn create_log_list( + logs: &ArrayDeque<[(Level, String); LOGS_BUFFER_SIZE], Wrapping>, + ) -> List<'_> { + let log_items = logs + .iter() + .map(|(level, log)| { + let style = match level { + Level::Debug => Style::default().fg(Color::Magenta), + Level::Error => Style::default().fg(Color::Red), + Level::Warn => Style::default().fg(Color::Yellow), + Level::Info => Style::default().fg(Color::Blue), + Level::Trace => Style::default(), + }; + + ListItem::new(Spans::from(vec![ + Span::styled(format!("{:<9}", level), style), + Span::raw(" "), + Span::raw(log), + ])) + }) + .collect::>(); + + List::new(log_items) + .block(Block::default().borders(Borders::ALL).title("Logs")) + .start_corner(Corner::BottomLeft) + } + async fn refresh_ui(ui_state: UiLoopState) -> Result { let mut logs = ui_state.logs; - let mut file_count_state = ui_state.file_count_state; let file_count = ui_state.file_count; let mut log_event_receiver = ui_state.log_event_receiver; let mut terminal = ui_state.terminal; - let rate = ui_state.coverage.rate.unwrap_or(0.0); - let features = ui_state.coverage.features.unwrap_or(0); - let covered = ui_state.coverage.covered.unwrap_or(0); + let rate = ui_state + .events + .get(&discriminant(&EventData::Rate(0.0))) + .and_then(|x| { + if let EventData::Rate(r) = x { + Some(*r) + } else { + None + } + }); + + let events = ui_state.events; Self::take_available_logs(&mut log_event_receiver, 10, &mut logs); terminal.draw(|f| { @@ -319,104 +460,39 @@ impl TerminalUi { let log_area = chunks[1]; let top_area = Layout::default() - .direction(Direction::Horizontal) + .direction(Direction::Vertical) .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref()) .split(chunks[0]); let file_count_area = top_area[0]; - let coverage_area = Layout::default() - .direction(Direction::Vertical) - .constraints([Constraint::Percentage(25), Constraint::Percentage(75)].as_ref()) - .margin(1) - .split(top_area[1]); - - let label = format!("{:.2}%", rate * 100.0); - - let coverage_block = Block::default().borders(Borders::ALL).title("Coverage:"); - - f.render_widget(coverage_block, top_area[1]); - - let gauge = Gauge::default() - .gauge_style( - Style::default() - .fg(Color::Magenta) - .bg(Color::Black) - .add_modifier(Modifier::ITALIC | Modifier::BOLD), - ) - .label(label) - .ratio(rate); - f.render_widget(gauge, coverage_area[0]); - - let coverage_info = List::new([ - ListItem::new(Spans::from(vec![ - Span::raw("features: "), - Span::raw(format!("{}", features)), - ])), - ListItem::new(Spans::from(vec![ - Span::raw("covered: "), - Span::raw(format!("{}", covered)), - ])), - ]); - - f.render_widget(coverage_info, coverage_area[1]); - - let mut sorted_file_count = file_count.iter().collect::>(); - - sorted_file_count.sort_by(|(p1, _), (p2, _)| p1.cmp(p2)); - let files = sorted_file_count - .iter() - .map(|(path, count)| { - ListItem::new(Spans::from(vec![ - Span::raw( - path.file_name() - .map(|f| f.to_string_lossy()) - .unwrap_or_default(), - ), - Span::raw(": "), - Span::raw(format!("{}", count)), - ])) - }) - .collect::>(); - - let log_list = List::new(files) - .block(Block::default().borders(Borders::ALL).title("files")) - .highlight_style(Style::default().add_modifier(Modifier::BOLD)) - .start_corner(Corner::TopLeft); - - f.render_stateful_widget(log_list, file_count_area, &mut file_count_state); - - let log_items = logs - .iter() - .map(|(level, log)| { - let style = match level { - Level::Debug => Style::default().fg(Color::Magenta), - Level::Error => Style::default().fg(Color::Red), - Level::Warn => Style::default().fg(Color::Yellow), - Level::Info => Style::default().fg(Color::Blue), - Level::Trace => Style::default(), - }; - - ListItem::new(Spans::from(vec![ - Span::styled(format!("{:<9}", level), style), - Span::raw(" "), - Span::raw(log), - ])) - }) - .collect::>(); + if let Some(rate) = rate { + let coverage_area = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Percentage(25), Constraint::Percentage(75)].as_ref()) + .split(top_area[1]); + + let gauge = Self::create_coverage_gauge(rate); + f.render_widget(gauge, coverage_area[0]); + let stats_paragraph = Self::create_stats_paragraph(&events); + f.render_widget(stats_paragraph, coverage_area[1]); + } else { + let stats_paragraph = Self::create_stats_paragraph(&events); + f.render_widget(stats_paragraph, top_area[1]); + } - let log_list = List::new(log_items) - .block(Block::default().borders(Borders::ALL).title("Logs")) - .start_corner(Corner::BottomLeft); + let file_count_paragraph = Self::create_file_count_paragraph(&file_count); + f.render_widget(file_count_paragraph, file_count_area); + let log_list = Self::create_log_list(&logs); f.render_widget(log_list, log_area); })?; Ok(UiLoopState { logs, - file_count_state, file_count, terminal, log_event_receiver, + events, ..ui_state }) } @@ -545,10 +621,14 @@ impl TerminalUi { ) .await } - TerminalEvent::Coverage(coverage) => Ok(UiLoopState { - coverage, - ..ui_state - }), + TerminalEvent::Telemetry(event_data) => { + let mut events = ui_state.events; + for e in event_data { + events.insert(discriminant(&e), e); + } + + Ok(UiLoopState { events, ..ui_state }) + } _ => Ok(ui_state), } }) From e0e18b53710b67964ef209f2956feba0f7a2b6c2 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 8 Apr 2021 09:57:53 -0700 Subject: [PATCH 6/7] fix telemetry event polling --- src/agent/onefuzz-agent/src/local/tui.rs | 31 ++++++++++-------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/tui.rs b/src/agent/onefuzz-agent/src/local/tui.rs index 4aefb0c87e..af27d73d93 100644 --- a/src/agent/onefuzz-agent/src/local/tui.rs +++ b/src/agent/onefuzz-agent/src/local/tui.rs @@ -2,7 +2,7 @@ // Licensed under the MIT License. use crate::local::common::UiEvent; -use anyhow::{Context, Result}; +use anyhow::Result; use crossterm::{ event::{self, Event, KeyCode}, execute, @@ -23,7 +23,7 @@ use std::{ }; use tokio::{ sync::{ - broadcast::{self, RecvError}, + broadcast::{self, TryRecvError}, mpsc::{self, UnboundedSender}, }, time::delay_for, @@ -65,6 +65,8 @@ impl From for UiLoopError { /// Maximum number of log message to display, arbitrarily chosen const LOGS_BUFFER_SIZE: usize = 100; const TICK_RATE: Duration = Duration::from_millis(250); +const FILE_MONITOR_POLLING_PERIOD: Duration = Duration::from_secs(5); +const EVENT_POLLING_PERIOD: Duration = Duration::from_secs(1); #[derive(Debug, Default)] struct CoverageData { @@ -234,7 +236,7 @@ impl TerminalUi { let mut rx = onefuzz_telemetry::subscribe_to_events(); while cancellation_rx.try_recv() == Err(broadcast::TryRecvError::Empty) { - match rx.recv().await { + match rx.try_recv() { Ok((_event, data)) => { let data = data .into_iter() @@ -242,9 +244,9 @@ impl TerminalUi { .collect::>(); let _ = ui_event_tx.send(TerminalEvent::Telemetry(data)); } - - Err(RecvError::Lagged(_)) => continue, - Err(RecvError::Closed) => break, + Err(TryRecvError::Empty) => delay_for(EVENT_POLLING_PERIOD).await, + Err(TryRecvError::Lagged(_)) => continue, + Err(TryRecvError::Closed) => break, } } Ok(()) @@ -270,7 +272,7 @@ impl TerminalUi { ) -> JoinHandle> { thread::spawn(move || { while Err(broadcast::TryRecvError::Empty) == cancellation_rx.try_recv() { - if event::poll(Duration::from_secs(1))? { + if event::poll(EVENT_POLLING_PERIOD)? { let event = event::read()?; if let Err(_err) = ui_event_tx.send(TerminalEvent::Input(event)) { return Ok(()); @@ -293,7 +295,7 @@ impl TerminalUi { break; } } - Err(mpsc::error::TryRecvError::Empty) => delay_for(Duration::from_secs(1)).await, + Err(mpsc::error::TryRecvError::Empty) => delay_for(EVENT_POLLING_PERIOD).await, Err(mpsc::error::TryRecvError::Closed) => break, } } @@ -429,7 +431,7 @@ impl TerminalUi { .collect::>(); List::new(log_items) - .block(Block::default().borders(Borders::ALL).title("Logs")) + .block(Block::default().borders(Borders::TOP).title("Logs")) .start_corner(Corner::BottomLeft) } @@ -543,14 +545,6 @@ impl TerminalUi { cancellation_tx: broadcast::Sender<()>, ) -> Result { let _ = cancellation_tx.send(()); - let file_monitors = ui_state.file_monitors.into_iter().map(|(_, v)| v).collect(); - tokio::time::timeout( - Duration::from_secs(10), - try_wait_all_join_handles(file_monitors), - ) - .await - .context("failed to close file monitoring tasks")? - .context("file monitoring task terminated with error")?; let mut terminal = ui_state.terminal; disable_raw_mode().map_err(|e| anyhow!("{:?}", e))?; execute!(terminal.backend_mut(), LeaveAlternateScreen).map_err(|e| anyhow!("{:?}", e))?; @@ -666,7 +660,8 @@ impl TerminalUi { { break; } - delay_for(Duration::from_secs(5)).await; + + delay_for(FILE_MONITOR_POLLING_PERIOD).await; } Ok(()) }) From 404ae681f7febb125e5bf8ffccd43d6d05456cd4 Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Fri, 9 Apr 2021 08:53:28 -0700 Subject: [PATCH 7/7] address PR comment --- src/agent/onefuzz-agent/src/local/generic_analysis.rs | 4 +--- src/agent/onefuzz-agent/src/local/generic_crash_report.rs | 4 +--- src/agent/onefuzz-agent/src/local/generic_generator.rs | 4 +--- src/agent/onefuzz-agent/src/local/libfuzzer.rs | 6 ++---- .../onefuzz-agent/src/local/libfuzzer_crash_report.rs | 4 +--- src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs | 6 ++---- src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs | 8 +++----- src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs | 6 ++---- src/agent/onefuzz-agent/src/local/radamsa.rs | 4 +--- src/agent/onefuzz-agent/src/local/test_input.rs | 7 +++---- 10 files changed, 17 insertions(+), 36 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/generic_analysis.rs b/src/agent/onefuzz-agent/src/local/generic_analysis.rs index a1dd17581a..4b0495512e 100644 --- a/src/agent/onefuzz-agent/src/local/generic_analysis.rs +++ b/src/agent/onefuzz-agent/src/local/generic_analysis.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_exe, get_hash_map, get_synced_dir, CmdType, - SyncCountDirMonitor, ANALYSIS_DIR, ANALYZER_ENV, ANALYZER_EXE, ANALYZER_OPTIONS, + SyncCountDirMonitor, UiEvent, ANALYSIS_DIR, ANALYZER_ENV, ANALYZER_EXE, ANALYZER_OPTIONS, CRASHES_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TOOLS_DIR, UNIQUE_REPORTS_DIR, }, @@ -18,8 +18,6 @@ use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - pub fn build_analysis_config( args: &clap::ArgMatches<'_>, input_queue: Option, diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 22c100d2bd..8039f4ed9a 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - SyncCountDirMonitor, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, + SyncCountDirMonitor, UiEvent, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_DEBUGGER, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, @@ -18,8 +18,6 @@ use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - pub fn build_report_config( args: &clap::ArgMatches<'_>, input_queue: Option, diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index b5589d2838..1f2bc4f055 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, - get_synced_dirs, CmdType, SyncCountDirMonitor, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, + get_synced_dirs, CmdType, SyncCountDirMonitor, UiEvent, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_DEBUGGER, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, READONLY_INPUTS, RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, TOOLS_DIR, @@ -18,8 +18,6 @@ use anyhow::Result; use clap::{App, Arg, SubCommand}; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - pub fn build_fuzz_config( args: &clap::ArgMatches<'_>, common: CommonConfig, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index 2f1589a73f..4b46c757d1 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -4,8 +4,8 @@ use crate::{ local::{ common::{ - build_local_context, wait_for_dir, DirectoryMonitorQueue, ANALYZER_EXE, COVERAGE_DIR, - REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR, + build_local_context, wait_for_dir, DirectoryMonitorQueue, UiEvent, ANALYZER_EXE, + COVERAGE_DIR, REGRESSION_REPORTS_DIR, UNIQUE_REPORTS_DIR, }, generic_analysis::{build_analysis_config, build_shared_args as build_analysis_args}, libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args}, @@ -28,8 +28,6 @@ use std::collections::HashSet; use tokio::{sync::mpsc::UnboundedSender, task::spawn}; use uuid::Uuid; -use super::common::UiEvent; - pub async fn run( args: &clap::ArgMatches<'_>, event_sender: Option>, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index 84f724c222..d217090694 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - SyncCountDirMonitor, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, CRASHES_DIR, + SyncCountDirMonitor, UiEvent, CHECK_FUZZER_HELP, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, @@ -18,8 +18,6 @@ use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - pub fn build_report_config( args: &clap::ArgMatches<'_>, input_queue: Option, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index 2ca0dbcabb..68d71c941a 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -4,8 +4,8 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - SyncCountDirMonitor, CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, - TARGET_OPTIONS, TARGET_WORKERS, + SyncCountDirMonitor, UiEvent, CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, + TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, }, tasks::{ config::CommonConfig, @@ -16,8 +16,6 @@ use anyhow::Result; use clap::{App, Arg, SubCommand}; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - const DISABLE_EXPECT_CRASH_ON_FAILURE: &str = "disable_expect_crash_on_failure"; pub fn build_fuzz_config( diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs index a72dc78b1b..9b8d91e1d4 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_merge.rs @@ -4,9 +4,9 @@ use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, - get_synced_dirs, CmdType, SyncCountDirMonitor, ANALYSIS_INPUTS, ANALYSIS_UNIQUE_INPUTS, - CHECK_FUZZER_HELP, INPUTS_DIR, PRESERVE_EXISTING_OUTPUTS, TARGET_ENV, TARGET_EXE, - TARGET_OPTIONS, + get_synced_dirs, CmdType, SyncCountDirMonitor, UiEvent, ANALYSIS_INPUTS, + ANALYSIS_UNIQUE_INPUTS, CHECK_FUZZER_HELP, INPUTS_DIR, PRESERVE_EXISTING_OUTPUTS, + TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, }, tasks::{ config::CommonConfig, @@ -18,8 +18,6 @@ use clap::{App, Arg, SubCommand}; use storage_queue::QueueClient; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - pub fn build_merge_config( args: &clap::ArgMatches<'_>, input_queue: Option, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs index 2e46cce097..4970a4192e 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_test_input.rs @@ -3,8 +3,8 @@ use crate::{ local::common::{ - build_local_context, get_cmd_arg, get_cmd_env, CmdType, CHECK_RETRY_COUNT, TARGET_ENV, - TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, + build_local_context, get_cmd_arg, get_cmd_env, CmdType, UiEvent, CHECK_RETRY_COUNT, + TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, }, tasks::report::libfuzzer_report::{test_input, TestInputArgs}, }; @@ -13,8 +13,6 @@ use clap::{App, Arg, SubCommand}; use std::path::PathBuf; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - pub async fn run( args: &clap::ArgMatches<'_>, event_sender: Option>, diff --git a/src/agent/onefuzz-agent/src/local/radamsa.rs b/src/agent/onefuzz-agent/src/local/radamsa.rs index 2447abe87c..44c5663a48 100644 --- a/src/agent/onefuzz-agent/src/local/radamsa.rs +++ b/src/agent/onefuzz-agent/src/local/radamsa.rs @@ -3,7 +3,7 @@ use crate::{ local::{ - common::{build_local_context, DirectoryMonitorQueue}, + common::{build_local_context, DirectoryMonitorQueue, UiEvent}, generic_crash_report::{build_report_config, build_shared_args as build_crash_args}, generic_generator::{build_fuzz_config, build_shared_args as build_fuzz_args}, }, @@ -17,8 +17,6 @@ use tokio::sync::mpsc::UnboundedSender; use tokio::task::spawn; use uuid::Uuid; -use super::common::UiEvent; - pub async fn run( args: &clap::ArgMatches<'_>, event_sender: Option>, diff --git a/src/agent/onefuzz-agent/src/local/test_input.rs b/src/agent/onefuzz-agent/src/local/test_input.rs index 9039ac33e1..225abdfca9 100644 --- a/src/agent/onefuzz-agent/src/local/test_input.rs +++ b/src/agent/onefuzz-agent/src/local/test_input.rs @@ -3,8 +3,9 @@ use crate::{ local::common::{ - build_local_context, get_cmd_arg, get_cmd_env, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, - DISABLE_CHECK_DEBUGGER, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, + build_local_context, get_cmd_arg, get_cmd_env, CmdType, UiEvent, CHECK_ASAN_LOG, + CHECK_RETRY_COUNT, DISABLE_CHECK_DEBUGGER, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, + TARGET_TIMEOUT, }, tasks::report::generic::{test_input, TestInputArgs}, }; @@ -13,8 +14,6 @@ use clap::{App, Arg, SubCommand}; use std::path::PathBuf; use tokio::sync::mpsc::UnboundedSender; -use super::common::UiEvent; - pub async fn run( args: &clap::ArgMatches<'_>, event_sender: Option>,