From 254f84d0632d5f9c3a4a2d4e90e5788e2fb95cc8 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Sun, 20 Dec 2020 17:40:51 -0500 Subject: [PATCH 01/30] support local fuzz management --- src/agent/onefuzz-agent/src/debug/cmd.rs | 41 ++++++----- .../src/debug/generic_crash_report.rs | 34 ++++----- .../src/debug/libfuzzer_coverage.rs | 33 ++++----- .../src/debug/libfuzzer_merge.rs | 36 ++++------ src/agent/onefuzz-agent/src/debug/mod.rs | 2 - src/agent/onefuzz-agent/src/local/cmd.rs | 29 ++++++++ src/agent/onefuzz-agent/src/local/common.rs | 49 +++++++++++++ .../libfuzzer_crash_report.rs | 71 ++++++++----------- .../src/{debug => local}/libfuzzer_fuzz.rs | 48 +++++-------- src/agent/onefuzz-agent/src/local/mod.rs | 7 ++ src/agent/onefuzz-agent/src/main.rs | 47 +++++++----- src/agent/onefuzz-agent/src/tasks/config.rs | 4 +- .../src/tasks/fuzz/libfuzzer_fuzz.rs | 43 +++++++---- .../src/tasks/report/libfuzzer_report.rs | 50 ++++++++++--- src/agent/onefuzz/src/libfuzzer.rs | 1 + 15 files changed, 301 insertions(+), 194 deletions(-) create mode 100644 src/agent/onefuzz-agent/src/local/cmd.rs create mode 100644 src/agent/onefuzz-agent/src/local/common.rs rename src/agent/onefuzz-agent/src/{debug => local}/libfuzzer_crash_report.rs (58%) rename src/agent/onefuzz-agent/src/{debug => local}/libfuzzer_fuzz.rs (67%) create mode 100644 src/agent/onefuzz-agent/src/local/mod.rs diff --git a/src/agent/onefuzz-agent/src/debug/cmd.rs b/src/agent/onefuzz-agent/src/debug/cmd.rs index 9d14c5327b..b4d2eea094 100644 --- a/src/agent/onefuzz-agent/src/debug/cmd.rs +++ b/src/agent/onefuzz-agent/src/debug/cmd.rs @@ -4,25 +4,34 @@ use anyhow::Result; use clap::{App, SubCommand}; -pub fn run(args: &clap::ArgMatches) -> Result<()> { +use crate::{ + debug::{generic_crash_report, libfuzzer_coverage, libfuzzer_merge}, + local::common::add_common_config, +}; + +const GENERIC_CRASH_REPORT: &str = "generic-crash-report"; +const LIBFUZZER_COVERAGE: &str = "libfuzzer-coverage"; +const LIBFUZZER_MERGE: &str = "libfuzzer-merge"; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { - ("generic-crash-report", Some(sub)) => crate::debug::generic_crash_report::run(sub)?, - ("libfuzzer-coverage", Some(sub)) => crate::debug::libfuzzer_coverage::run(sub)?, - ("libfuzzer-crash-report", Some(sub)) => crate::debug::libfuzzer_crash_report::run(sub)?, - ("libfuzzer-fuzz", Some(sub)) => crate::debug::libfuzzer_fuzz::run(sub)?, - ("libfuzzer-merge", Some(sub)) => crate::debug::libfuzzer_merge::run(sub)?, - _ => println!("missing subcommand\nUSAGE : {}", args.usage()), + (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub).await, + (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, + (LIBFUZZER_MERGE, Some(sub)) => libfuzzer_merge::run(sub).await, + _ => { + anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); + } } - - Ok(()) } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("debug") +pub fn args(name: &str) -> App<'static, 'static> { + SubCommand::with_name(name) .about("unsupported internal debugging commands") - .subcommand(crate::debug::generic_crash_report::args()) - .subcommand(crate::debug::libfuzzer_coverage::args()) - .subcommand(crate::debug::libfuzzer_crash_report::args()) - .subcommand(crate::debug::libfuzzer_fuzz::args()) - .subcommand(crate::debug::libfuzzer_merge::args()) + .subcommand(add_common_config(generic_crash_report::args( + GENERIC_CRASH_REPORT, + ))) + .subcommand(add_common_config(libfuzzer_coverage::args( + LIBFUZZER_COVERAGE, + ))) + .subcommand(add_common_config(libfuzzer_merge::args(LIBFUZZER_MERGE))) } diff --git a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs b/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs index 016657c139..85f98e261e 100644 --- a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::tasks::{ - config::CommonConfig, - report::generic::{Config, GenericReportProcessor}, - utils::parse_key_value, +use crate::{ + local::common::build_common_config, + tasks::{ + report::generic::{Config, GenericReportProcessor}, + utils::parse_key_value, + }, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; @@ -13,9 +15,7 @@ use std::{ collections::HashMap, path::{Path, PathBuf}, }; -use tokio::runtime::Runtime; use url::Url; -use uuid::Uuid; async fn run_impl(input: String, config: Config) -> Result<()> { let input_path = Path::new(&input); @@ -27,7 +27,7 @@ async fn run_impl(input: String, config: Config) -> Result<()> { Ok(()) } -pub fn run(args: &clap::ArgMatches) -> Result<()> { +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let target_exe = value_t!(args, "target_exe", PathBuf)?; let input = value_t!(args, "input", String)?; let target_timeout = value_t!(args, "target_timeout", u64).ok(); @@ -42,6 +42,8 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { target_env.insert(k, v); } + let common = build_common_config(args)?; + let config = Config { target_exe, target_env, @@ -58,24 +60,14 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { path: "unique_reports".into(), url: BlobContainerUrl::new(url::Url::parse("https://contoso.com/unique_reports")?)?, }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - }, + common, }; - let mut rt = Runtime::new()?; - rt.block_on(async { run_impl(input, config).await })?; - - Ok(()) + run_impl(input, config).await } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("generic-crash-report") +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) .about("execute a local-only generic crash report") .arg( Arg::with_name("target_exe") diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs index c2c76dbbc0..fd8c0b8f7c 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::tasks::{ - config::CommonConfig, - coverage::libfuzzer_coverage::{Config, CoverageProcessor}, - utils::parse_key_value, +use crate::{ + local::common::build_common_config, + tasks::{ + coverage::libfuzzer_coverage::{Config, CoverageProcessor}, + utils::parse_key_value, + }, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; @@ -14,9 +16,7 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use tokio::runtime::Runtime; use url::Url; -use uuid::Uuid; async fn run_impl(input: String, config: Config) -> Result<()> { let mut processor = CoverageProcessor::new(Arc::new(config)) @@ -36,7 +36,7 @@ async fn run_impl(input: String, config: Config) -> Result<()> { Ok(()) } -pub fn run(args: &clap::ArgMatches) -> Result<()> { +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let target_exe = value_t!(args, "target_exe", PathBuf)?; let input = value_t!(args, "input", String)?; let result_dir = value_t!(args, "result_dir", String)?; @@ -48,6 +48,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { target_env.insert(k, v); } + let common = build_common_config(args)?; let config = Config { target_exe, target_env, @@ -58,24 +59,14 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { path: result_dir.into(), url: BlobContainerUrl::new(Url::parse("https://contoso.com/coverage")?)?, }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - }, + common, }; - let mut rt = Runtime::new()?; - rt.block_on(run_impl(input, config))?; - - Ok(()) + run_impl(input, config).await } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-coverage") +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) .about("execute a local-only libfuzzer coverage task") .arg( Arg::with_name("target_exe") diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs index 94fd3f2d60..929649fa23 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs @@ -1,20 +1,20 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::tasks::{ - config::CommonConfig, - merge::libfuzzer_merge::{merge_inputs, Config}, - utils::parse_key_value, +use crate::{ + local::common::build_common_config, + tasks::{ + merge::libfuzzer_merge::{merge_inputs, Config}, + utils::parse_key_value, + }, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; use std::{collections::HashMap, path::PathBuf, sync::Arc}; -use tokio::runtime::Runtime; use url::Url; -use uuid::Uuid; -pub fn run(args: &clap::ArgMatches) -> Result<()> { +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let target_exe = value_t!(args, "target_exe", PathBuf)?; let inputs = value_t!(args, "inputs", String)?; let unique_inputs = value_t!(args, "unique_inputs", String)?; @@ -26,6 +26,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { target_env.insert(k, v); } + let common = build_common_config(args)?; let config = Arc::new(Config { target_exe, target_env, @@ -39,28 +40,17 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { path: unique_inputs.into(), url: BlobContainerUrl::new(Url::parse("https://contoso.com/unique_inputs")?)?, }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - }, + common, preserve_existing_outputs: true, }); - let mut rt = Runtime::new()?; - rt.block_on(merge_inputs( - config.clone(), - vec![config.clone().inputs[0].path.clone()], - ))?; - + let results = merge_inputs(config.clone(), vec![config.clone().inputs[0].path.clone()]).await?; + println!("{:#?}", results); Ok(()) } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-merge") +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) .about("execute a local-only libfuzzer merge task") .arg( Arg::with_name("target_exe") diff --git a/src/agent/onefuzz-agent/src/debug/mod.rs b/src/agent/onefuzz-agent/src/debug/mod.rs index d84cc95080..be0d0e0016 100644 --- a/src/agent/onefuzz-agent/src/debug/mod.rs +++ b/src/agent/onefuzz-agent/src/debug/mod.rs @@ -4,6 +4,4 @@ pub mod cmd; pub mod generic_crash_report; pub mod libfuzzer_coverage; -pub mod libfuzzer_crash_report; -pub mod libfuzzer_fuzz; pub mod libfuzzer_merge; diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs new file mode 100644 index 0000000000..fe61be3ef5 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use anyhow::Result; +use clap::{App, SubCommand}; + +use crate::local::{common::add_common_config, libfuzzer_crash_report, libfuzzer_fuzz}; + +const LIBFUZZER_FUZZ: &str = "libfuzzer-fuzz"; +const LIBFUZZER_CRASH_REPORT: &str = "libfuzzer-crash-report"; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + match args.subcommand() { + (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, + (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, + _ => { + anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); + } + } +} + +pub fn args(name: &str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("pre-release local fuzzing") + .subcommand(add_common_config(libfuzzer_fuzz::args(LIBFUZZER_FUZZ))) + .subcommand(add_common_config(libfuzzer_crash_report::args( + LIBFUZZER_CRASH_REPORT, + ))) +} diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs new file mode 100644 index 0000000000..3229f77a17 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -0,0 +1,49 @@ +use crate::tasks::config::CommonConfig; +use anyhow::Result; +use clap::{App, Arg, ArgMatches}; +use uuid::Uuid; + +pub fn add_common_config(app: App<'static, 'static>) -> App<'static, 'static> { + app.arg( + Arg::with_name("job_id") + .long("job_id") + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name("task_id") + .long("task_id") + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name("instance_id") + .long("instance_id") + .takes_value(true) + .required(false), + ) +} + +fn get_uuid(name: &str, args: &ArgMatches<'_>) -> Result { + match value_t!(args, name, String) { + Ok(x) => Uuid::parse_str(&x) + .map_err(|x| format_err!("invalid {}. uuid expected. {})", name, x)), + Err(_) => Ok(Uuid::nil()), + } +} + +pub fn build_common_config(args: &ArgMatches<'_>) -> Result { + let job_id = get_uuid("job_id", args)?; + let task_id = get_uuid("task_id", args)?; + let instance_id = get_uuid("instance_id", args)?; + + let config = CommonConfig { + heartbeat_queue: None, + instrumentation_key: None, + telemetry_key: None, + job_id, + task_id, + instance_id, + }; + Ok(config) +} diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs similarity index 58% rename from src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs rename to src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index d6a16b111e..da89802130 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -1,36 +1,24 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::tasks::{ - config::CommonConfig, - report::libfuzzer_report::{AsanProcessor, Config}, - utils::parse_key_value, +use crate::{ + local::common::build_common_config, + tasks::{ + report::libfuzzer_report::{Config, ReportTask}, + utils::parse_key_value, + }, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; -use tokio::runtime::Runtime; +use std::{collections::HashMap, path::PathBuf}; use url::Url; -use uuid::Uuid; - -async fn run_impl(input: String, config: Config) -> Result<()> { - let task = AsanProcessor::new(Arc::new(config)).await?; - - let test_url = Url::parse("https://contoso.com/sample-container/blob.txt")?; - let input_path = Path::new(&input); - let result = task.test_input(test_url, &input_path).await; - println!("{:#?}", result); - Ok(()) -} -pub fn run(args: &clap::ArgMatches) -> Result<()> { +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let target_exe = value_t!(args, "target_exe", PathBuf)?; - let input = value_t!(args, "input", String)?; + let crashes_dir = value_t!(args, "crashes_dir", PathBuf)?; + let reports_dir = value_t!(args, "reports_dir", PathBuf)?; + let target_options = args.values_of_lossy("target_options").unwrap_or_default(); let mut target_env = HashMap::new(); for opt in args.values_of_lossy("target_env").unwrap_or_default() { @@ -40,6 +28,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { let target_timeout = value_t!(args, "target_timeout", u64).ok(); let check_retry_count = value_t!(args, "check_retry_count", u64)?; + let common = build_common_config(args)?; let config = Config { target_exe, target_env, @@ -47,38 +36,40 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { target_timeout, check_retry_count, input_queue: None, - crashes: None, + crashes: Some(SyncedDir { + path: crashes_dir, + url: BlobContainerUrl::new(Url::parse("https://contoso.com/crashes")?)?, + }), reports: None, no_repro: None, unique_reports: SyncedDir { - path: "unique_reports".into(), + path: reports_dir, url: BlobContainerUrl::new(Url::parse("https://contoso.com/unique_reports")?)?, }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - }, + common, }; - let mut rt = Runtime::new()?; - rt.block_on(async { run_impl(input, config).await })?; - - Ok(()) + ReportTask::new(config).run_local().await } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-crash-report") +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) .about("execute a local-only libfuzzer crash report task") .arg( Arg::with_name("target_exe") .takes_value(true) .required(true), ) - .arg(Arg::with_name("input").takes_value(true).required(true)) + .arg( + Arg::with_name("crashes_dir") + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name("reports_dir") + .takes_value(true) + .required(true), + ) .arg( Arg::with_name("target_env") .long("target_env") diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs similarity index 67% rename from src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs rename to src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index b126c7ae86..0c4b113f4e 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -1,31 +1,25 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::tasks::{ - config::CommonConfig, - fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, - utils::parse_key_value, +use crate::{ + local::common::build_common_config, + tasks::{ + fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, + utils::parse_key_value, + }, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; use std::{collections::HashMap, path::PathBuf}; -use tokio::runtime::Runtime; use url::Url; -use uuid::Uuid; -async fn run_impl(config: Config) -> Result<()> { - let fuzzer = LibFuzzerFuzzTask::new(config)?; - let result = fuzzer.start_fuzzer_monitor(0, None).await?; - println!("{:#?}", result); - Ok(()) -} - -pub fn run(args: &clap::ArgMatches) -> Result<()> { +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let crashes_dir = value_t!(args, "crashes_dir", String)?; let inputs_dir = value_t!(args, "inputs_dir", String)?; let target_exe = value_t!(args, "target_exe", PathBuf)?; let target_options = args.values_of_lossy("target_options").unwrap_or_default(); + let target_workers = value_t!(args, "target_workers", u64).unwrap_or_default(); let mut target_env = HashMap::new(); for opt in args.values_of_lossy("target_env").unwrap_or_default() { let (k, v) = parse_key_value(opt)?; @@ -33,7 +27,6 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { } let readonly_inputs = None; - let target_workers = Some(1); let inputs = SyncedDir { path: inputs_dir.into(), @@ -46,7 +39,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { }; let ensemble_sync_delay = None; - + let common = build_common_config(args)?; let config = Config { inputs, readonly_inputs, @@ -56,24 +49,14 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { target_options, target_workers, ensemble_sync_delay, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - }, + common, }; - let mut rt = Runtime::new()?; - rt.block_on(async { run_impl(config).await })?; - - Ok(()) + LibFuzzerFuzzTask::new(config)?.local_run().await } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-fuzz") +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) .about("execute a local-only libfuzzer crash report task") .arg( Arg::with_name("target_exe") @@ -104,4 +87,9 @@ pub fn args() -> App<'static, 'static> { .takes_value(true) .required(true), ) + .arg( + Arg::with_name("target_workers") + .long("target_workers") + .takes_value(true), + ) } diff --git a/src/agent/onefuzz-agent/src/local/mod.rs b/src/agent/onefuzz-agent/src/local/mod.rs new file mode 100644 index 0000000000..049c4240dd --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +pub mod cmd; +pub mod common; +pub mod libfuzzer_crash_report; +pub mod libfuzzer_fuzz; diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 129651a735..455f8641cb 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -13,16 +13,21 @@ extern crate clap; use std::path::PathBuf; use anyhow::Result; -use clap::{App, Arg, SubCommand}; +use clap::{App, Arg, ArgMatches, SubCommand}; use onefuzz::telemetry::{self}; +use std::io::{stdout, Write}; mod debug; +mod local; mod tasks; -use tasks::config::Config; +use tasks::config::{CommonConfig, Config}; + +const LOCAL_CMD: &str = "local"; +const DEBUG_CMD: &str = "debug"; fn main() -> Result<()> { - env_logger::init(); + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); let built_version = format!( "{} onefuzz:{} git:{}", @@ -39,16 +44,23 @@ fn main() -> Result<()> { .short("c") .takes_value(true), ) - .subcommand(debug::cmd::args()) + .subcommand(local::cmd::args(LOCAL_CMD)) + .subcommand(debug::cmd::args(DEBUG_CMD)) .subcommand(SubCommand::with_name("licenses").about("display third-party licenses")); let matches = app.get_matches(); + let mut rt = tokio::runtime::Runtime::new()?; + rt.block_on(run(matches)) +} + +async fn run(matches: ArgMatches<'_>) -> Result<()> { match matches.subcommand() { ("licenses", Some(_)) => { return licenses(); } - ("debug", Some(sub)) => return crate::debug::cmd::run(sub), + (DEBUG_CMD, Some(sub)) => return debug::cmd::run(sub).await, + (LOCAL_CMD, Some(sub)) => return local::cmd::run(sub).await, _ => {} // no subcommand } @@ -57,34 +69,31 @@ fn main() -> Result<()> { return Ok(()); } - let config_path: PathBuf = matches.value_of("config").unwrap().parse()?; - let config = Config::from_file(config_path)?; + let config_arg = matches.value_of("config").unwrap(); + run_config(config_arg).await +} - init_telemetry(&config); +async fn run_config(config_arg: &str) -> Result<()> { + let config_path: PathBuf = config_arg.parse()?; + let config = Config::from_file(config_path)?; + init_telemetry(config.common()); verbose!("config parsed"); - - let mut rt = tokio::runtime::Runtime::new()?; - - let result = rt.block_on(config.run()); + let result = config.run().await; if let Err(err) = &result { error!("error running task: {}", err); } telemetry::try_flush_and_close(); - result } fn licenses() -> Result<()> { - use std::io::{self, Write}; - io::stdout().write_all(include_bytes!("../../data/licenses.json"))?; + stdout().write_all(include_bytes!("../../data/licenses.json"))?; Ok(()) } -fn init_telemetry(config: &Config) { - let inst_key = config.common().instrumentation_key; - let tele_key = config.common().telemetry_key; - telemetry::set_appinsights_clients(inst_key, tele_key); +fn init_telemetry(config: &CommonConfig) { + telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key); } diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 03e6ace359..b01d53b76c 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -141,12 +141,12 @@ impl Config { match self { Config::LibFuzzerFuzz(config) => { fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask::new(config)? - .start() + .managed_run() .await } Config::LibFuzzerReport(config) => { report::libfuzzer_report::ReportTask::new(config) - .run() + .run_managed() .await } Config::LibFuzzerCoverage(config) => { diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index 7883ed8afb..08a9909fa2 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -36,6 +36,11 @@ const PROC_INFO_PERIOD: Duration = Duration::from_secs(30); // Period of reporting fuzzer-generated runtime stats. const RUNTIME_STATS_PERIOD: Duration = Duration::from_secs(60); +pub fn default_workers() -> u64 { + let cpus = num_cpus::get() as u64; + u64::max(1, cpus - 1) +} + #[derive(Debug, Deserialize, Clone)] pub struct Config { pub inputs: SyncedDir, @@ -44,7 +49,9 @@ pub struct Config { pub target_exe: PathBuf, pub target_env: HashMap, pub target_options: Vec, - pub target_workers: Option, + + #[serde(default = "default_workers")] + pub target_workers: u64, pub ensemble_sync_delay: Option, #[serde(flatten)] @@ -60,13 +67,21 @@ impl LibFuzzerFuzzTask { Ok(Self { config }) } - pub async fn start(&self) -> Result<()> { - let workers = self.config.target_workers.unwrap_or_else(|| { - let cpus = num_cpus::get() as u64; - u64::max(1, cpus - 1) - }); + fn workers(&self) -> u64 { + match self.config.target_workers { + 0 => default_workers(), + x => x, + } + } + + pub async fn local_run(&self) -> Result<()> { + self.init_directories().await?; + self.run_fuzzers(None).await + } + pub async fn managed_run(&self) -> Result<()> { self.init_directories().await?; + let hb_client = self.config.common.init_heartbeat().await?; // To be scheduled. @@ -75,15 +90,19 @@ impl LibFuzzerFuzzTask { let new_crashes = self.config.crashes.monitor_results(new_result); let (stats_sender, stats_receiver) = mpsc::unbounded_channel(); - let report_stats = report_runtime_stats(workers as usize, stats_receiver, hb_client); + let report_stats = report_runtime_stats(self.workers() as usize, stats_receiver, hb_client); + let fuzzers = self.run_fuzzers(Some(&stats_sender)); + futures::try_join!(resync, new_inputs, new_crashes, fuzzers, report_stats)?; - let fuzzers: Vec<_> = (0..workers) - .map(|id| self.start_fuzzer_monitor(id, Some(&stats_sender))) - .collect(); + Ok(()) + } - let fuzzers = try_join_all(fuzzers); + pub async fn run_fuzzers(&self, stats_sender: Option<&StatsSender>) -> Result<()> { + let fuzzers: Vec<_> = (0..self.workers()) + .map(|id| self.start_fuzzer_monitor(id, stats_sender)) + .collect(); - futures::try_join!(resync, new_inputs, new_crashes, fuzzers, report_stats)?; + try_join_all(fuzzers).await?; Ok(()) } diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index 0eb2226829..ef4f96fc2e 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -5,7 +5,10 @@ use super::crash_report::*; use crate::tasks::{config::CommonConfig, generic::input_poller::*, heartbeat::*}; use anyhow::Result; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, libfuzzer::LibFuzzer, sha256, syncdir::SyncedDir}; +use futures::stream::StreamExt; +use onefuzz::{ + blob::BlobUrl, libfuzzer::LibFuzzer, monitor::DirectoryMonitor, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -15,6 +18,10 @@ use std::{ }; use storage_queue::Message; +fn default_bool_true() -> bool { + true +} + #[derive(Debug, Deserialize)] pub struct Config { pub target_exe: PathBuf, @@ -30,6 +37,9 @@ pub struct Config { #[serde(default)] pub check_retry_count: u64, + #[serde(default = "default_bool_true")] + pub check_queue: bool, + #[serde(flatten)] pub common: CommonConfig, } @@ -40,16 +50,38 @@ pub struct ReportTask { } impl ReportTask { - pub fn new(config: impl Into>) -> Self { - let config = config.into(); - + pub fn new(config: Config) -> Self { let working_dir = config.common.task_id.to_string(); let poller = InputPoller::new(working_dir); + let config = Arc::new(config); Self { config, poller } } - pub async fn run(&mut self) -> Result<()> { + pub async fn run_local(&mut self) -> Result<()> { + let mut processor = AsanProcessor::new(self.config.clone()).await?; + let crashes = match &self.config.crashes { + Some(x) => x, + None => bail!("missing crashes directory"), + }; + + self.poller.batch_process(&mut processor, crashes).await?; + + if self.config.check_queue { + let mut monitor = DirectoryMonitor::new(crashes.path.clone()); + monitor.start()?; + + while let Some(crash) = monitor.next().await { + let test_url = Url::parse("https://contoso.com/sample-container/blob.txt")?; + let input_path = Path::new(&crash); + let result = processor.test_input(test_url, &input_path).await?; + } + } + + Ok(()) + } + + pub async fn run_managed(&mut self) -> Result<()> { info!("Starting libFuzzer crash report task"); let mut processor = AsanProcessor::new(self.config.clone()).await?; @@ -57,9 +89,11 @@ impl ReportTask { self.poller.batch_process(&mut processor, crashes).await?; } - if let Some(queue) = &self.config.input_queue { - let callback = CallbackImpl::new(queue.clone(), processor); - self.poller.run(callback).await?; + if self.config.check_queue { + if let Some(queue) = &self.config.input_queue { + let callback = CallbackImpl::new(queue.clone(), processor); + self.poller.run(callback).await?; + } } Ok(()) } diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index 66e9e14231..0cb811f30b 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -16,6 +16,7 @@ use tokio::process::{Child, Command}; const DEFAULT_MAX_TOTAL_SECONDS: i32 = 10 * 60; +#[derive(Debug)] pub struct LibFuzzerMergeOutput { pub added_files_count: i32, pub added_feature_count: i32, From 9d2d947705b7e9623112c3833dc45945c0c9591c Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Sun, 20 Dec 2020 22:05:47 -0500 Subject: [PATCH 02/30] continued dev --- .../src/debug/generic_crash_report.rs | 6 +- .../src/debug/libfuzzer_coverage.rs | 5 +- .../src/debug/libfuzzer_merge.rs | 7 +- src/agent/onefuzz-agent/src/local/cmd.rs | 5 +- src/agent/onefuzz-agent/src/local/common.rs | 54 ++++++++ .../onefuzz-agent/src/local/libfuzzer.rs | 79 ++++++++++++ .../src/local/libfuzzer_crash_report.rs | 117 ++++++++++-------- .../onefuzz-agent/src/local/libfuzzer_fuzz.rs | 81 ++++-------- src/agent/onefuzz-agent/src/local/mod.rs | 1 + .../src/tasks/analysis/generic.rs | 5 +- src/agent/onefuzz-agent/src/tasks/config.rs | 2 +- .../src/tasks/coverage/libfuzzer_coverage.rs | 2 +- .../src/tasks/generic/input_poller.rs | 17 ++- .../tasks/generic/input_poller/callback.rs | 2 +- .../src/tasks/report/crash_report.rs | 44 +++++-- .../onefuzz-agent/src/tasks/report/generic.rs | 13 +- .../src/tasks/report/libfuzzer_report.rs | 52 ++++++-- src/agent/onefuzz/src/syncdir.rs | 36 +++++- 18 files changed, 368 insertions(+), 160 deletions(-) create mode 100644 src/agent/onefuzz-agent/src/local/libfuzzer.rs diff --git a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs b/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs index 85f98e261e..68a391e208 100644 --- a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs @@ -10,7 +10,7 @@ use crate::{ }; use anyhow::Result; use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; +use onefuzz::syncdir::SyncedDir; use std::{ collections::HashMap, path::{Path, PathBuf}, @@ -22,7 +22,7 @@ async fn run_impl(input: String, config: Config) -> Result<()> { let test_url = Url::parse("https://contoso.com/sample-container/blob.txt")?; let heartbeat_client = config.common.init_heartbeat().await?; let processor = GenericReportProcessor::new(&config, heartbeat_client); - let result = processor.test_input(test_url, input_path).await?; + let result = processor.test_input(Some(test_url), input_path).await?; println!("{:#?}", result); Ok(()) } @@ -58,7 +58,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { reports: None, unique_reports: SyncedDir { path: "unique_reports".into(), - url: BlobContainerUrl::new(url::Url::parse("https://contoso.com/unique_reports")?)?, + url: None, }, common, }; diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs index fd8c0b8f7c..a8ed2ce2e2 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs @@ -10,13 +10,12 @@ use crate::{ }; use anyhow::Result; use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; +use onefuzz::syncdir::SyncedDir; use std::{ collections::HashMap, path::{Path, PathBuf}, sync::Arc, }; -use url::Url; async fn run_impl(input: String, config: Config) -> Result<()> { let mut processor = CoverageProcessor::new(Arc::new(config)) @@ -57,7 +56,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { readonly_inputs: vec![], coverage: SyncedDir { path: result_dir.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/coverage")?)?, + url: None, }, common, }; diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs index 929649fa23..faaae3f800 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs @@ -10,9 +10,8 @@ use crate::{ }; use anyhow::Result; use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; +use onefuzz::syncdir::SyncedDir; use std::{collections::HashMap, path::PathBuf, sync::Arc}; -use url::Url; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let target_exe = value_t!(args, "target_exe", PathBuf)?; @@ -34,11 +33,11 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { input_queue: None, inputs: vec![SyncedDir { path: inputs.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/inputs")?)?, + url: None, }], unique_inputs: SyncedDir { path: unique_inputs.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/unique_inputs")?)?, + url: None, }, common, preserve_existing_outputs: true, diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs index fe61be3ef5..5b7c3b31f9 100644 --- a/src/agent/onefuzz-agent/src/local/cmd.rs +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -4,13 +4,15 @@ use anyhow::Result; use clap::{App, SubCommand}; -use crate::local::{common::add_common_config, libfuzzer_crash_report, libfuzzer_fuzz}; +use crate::local::{common::add_common_config, libfuzzer, libfuzzer_crash_report, libfuzzer_fuzz}; +const LIBFUZZER: &str = "libfuzzer"; const LIBFUZZER_FUZZ: &str = "libfuzzer-fuzz"; const LIBFUZZER_CRASH_REPORT: &str = "libfuzzer-crash-report"; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { + (LIBFUZZER, Some(sub)) => libfuzzer::run(sub).await, (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, _ => { @@ -22,6 +24,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub fn args(name: &str) -> App<'static, 'static> { SubCommand::with_name(name) .about("pre-release local fuzzing") + .subcommand(add_common_config(libfuzzer::args(LIBFUZZER))) .subcommand(add_common_config(libfuzzer_fuzz::args(LIBFUZZER_FUZZ))) .subcommand(add_common_config(libfuzzer_crash_report::args( LIBFUZZER_CRASH_REPORT, diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 3229f77a17..1036ff8503 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -1,8 +1,62 @@ use crate::tasks::config::CommonConfig; +use crate::tasks::utils::parse_key_value; use anyhow::Result; use clap::{App, Arg, ArgMatches}; +use std::collections::HashMap; use uuid::Uuid; +pub const TARGET_EXE: &str = "target_exe"; +const TARGET_ENV: &str = "target_env"; +pub const TARGET_OPTIONS: &str = "target_options"; +pub const INPUTS_DIR: &str = "inputs_dir"; +pub const CRASHES_DIR: &str = "crashes_dir"; +pub const TARGET_WORKERS: &str = "target_workers"; +pub const REPORTS_DIR: &str = "reports_dir"; +pub const NO_REPRO_DIR: &str = "no_repro_dir"; +pub const TARGET_TIMEOUT: &str = "target_timeout"; +pub const CHECK_RETRY_COUNT: &str = "check_retry_count"; +pub const DISABLE_CHECK_QUEUE: &str = "disable_check_queue"; +pub const UNIQUE_REPORTS_DIR: &str = "unique_reports_dir"; + +pub fn add_target_cmd_options( + exe: bool, + arg: bool, + env: bool, + mut app: App<'static, 'static>, +) -> App<'static, 'static> { + if exe { + app = app.arg(Arg::with_name(TARGET_EXE).takes_value(true).required(true)); + } + if arg { + app = app.arg( + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + ) + } + if env { + app = app.arg( + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set target_env first"), + ) + } + app +} + +pub fn get_target_env(args: &clap::ArgMatches<'_>) -> Result> { + let mut target_env = HashMap::new(); + for opt in args.values_of_lossy("target_env").unwrap_or_default() { + let (k, v) = parse_key_value(opt)?; + target_env.insert(k, v); + } + Ok(target_env) +} + pub fn add_common_config(app: App<'static, 'static>) -> App<'static, 'static> { app.arg( Arg::with_name("job_id") diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs new file mode 100644 index 0000000000..871f78b0ee --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::{ + common::{ + add_target_cmd_options, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, + INPUTS_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, + UNIQUE_REPORTS_DIR, + }, + libfuzzer_crash_report::build_report_config, + libfuzzer_fuzz::build_fuzz_config, + }, + tasks::{fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask, report::libfuzzer_report::ReportTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let fuzz_config = build_fuzz_config(args)?; + let report_config = build_report_config(args)?; + + let fuzzer = LibFuzzerFuzzTask::new(fuzz_config)?; + let fuzz_task = fuzzer.local_run(); + + let report = ReportTask::new(report_config); + let report_task = report.local_run(); + + tokio::try_join!(fuzz_task, report_task)?; + + Ok(()) +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("run a local libfuzzer & crash reporting task"); + + app = add_target_cmd_options(true, true, true, app); + + app.arg(Arg::with_name(INPUTS_DIR).takes_value(true).required(true)) + .arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) + .arg( + Arg::with_name(TARGET_WORKERS) + .long(TARGET_WORKERS) + .takes_value(true), + ) + .arg( + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name(UNIQUE_REPORTS_DIR) + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT), + ) + .arg( + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + ) + .arg( + Arg::with_name(DISABLE_CHECK_QUEUE) + .takes_value(false) + .long(DISABLE_CHECK_QUEUE), + ) +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index da89802130..ee87459d2e 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -2,31 +2,39 @@ // Licensed under the MIT License. use crate::{ - local::common::build_common_config, - tasks::{ - report::libfuzzer_report::{Config, ReportTask}, - utils::parse_key_value, + local::common::{ + add_target_cmd_options, build_common_config, get_target_env, CHECK_RETRY_COUNT, + CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_EXE, TARGET_OPTIONS, + TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, + tasks::report::libfuzzer_report::{Config, ReportTask}, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{collections::HashMap, path::PathBuf}; -use url::Url; +use std::path::PathBuf; -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let crashes_dir = value_t!(args, "crashes_dir", PathBuf)?; - let reports_dir = value_t!(args, "reports_dir", PathBuf)?; +pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { + let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; + let crashes = Some(value_t!(args, CRASHES_DIR, PathBuf)?.into()); + let reports = if args.is_present(REPORTS_DIR) { + Some(value_t!(args, REPORTS_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + let no_repro = if args.is_present(NO_REPRO_DIR) { + Some(value_t!(args, NO_REPRO_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + + let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); + + let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); + let target_env = get_target_env(args)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - let target_timeout = value_t!(args, "target_timeout", u64).ok(); - let check_retry_count = value_t!(args, "check_retry_count", u64)?; + let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); + let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; + let check_queue = !args.is_present(DISABLE_CHECK_QUEUE); let common = build_common_config(args)?; let config = Config { @@ -36,63 +44,62 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { target_timeout, check_retry_count, input_queue: None, - crashes: Some(SyncedDir { - path: crashes_dir, - url: BlobContainerUrl::new(Url::parse("https://contoso.com/crashes")?)?, - }), - reports: None, - no_repro: None, - unique_reports: SyncedDir { - path: reports_dir, - url: BlobContainerUrl::new(Url::parse("https://contoso.com/unique_reports")?)?, - }, + check_queue, + crashes, + reports, + no_repro, + unique_reports, common, }; + Ok(config) +} - ReportTask::new(config).run_local().await +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_report_config(args)?; + ReportTask::new(config).local_run().await } -pub fn args(name: &'static str) -> App<'static, 'static> { - SubCommand::with_name(name) - .about("execute a local-only libfuzzer crash report task") +pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { + app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) .arg( - Arg::with_name("target_exe") + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) .takes_value(true) - .required(true), + .required(false), ) .arg( - Arg::with_name("crashes_dir") + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) .takes_value(true) - .required(true), + .required(false), ) .arg( - Arg::with_name("reports_dir") + Arg::with_name(UNIQUE_REPORTS_DIR) .takes_value(true) .required(true), ) .arg( - Arg::with_name("target_env") - .long("target_env") + Arg::with_name(TARGET_TIMEOUT) .takes_value(true) - .multiple(true), + .long(TARGET_TIMEOUT), ) .arg( - Arg::with_name("target_options") - .long("target_options") + Arg::with_name(CHECK_RETRY_COUNT) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set target_env first"), - ) - .arg( - Arg::with_name("target_timeout") - .takes_value(true) - .long("target_timeout"), + .long(CHECK_RETRY_COUNT) + .default_value("0"), ) .arg( - Arg::with_name("check_retry_count") - .takes_value(true) - .long("check_retry_count") - .default_value("0"), + Arg::with_name(DISABLE_CHECK_QUEUE) + .takes_value(false) + .long(DISABLE_CHECK_QUEUE), ) } + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = + SubCommand::with_name(name).about("execute a local-only libfuzzer crash report task"); + + app = add_target_cmd_options(true, true, true, app); + add_report_options(app) +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index 0c4b113f4e..dd3eb1af0e 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -2,42 +2,25 @@ // Licensed under the MIT License. use crate::{ - local::common::build_common_config, - tasks::{ - fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, - utils::parse_key_value, + local::common::{ + add_target_cmd_options, build_common_config, get_target_env, CRASHES_DIR, INPUTS_DIR, + TARGET_WORKERS, }, + tasks::fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{collections::HashMap, path::PathBuf}; -use url::Url; +use std::path::PathBuf; -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let crashes_dir = value_t!(args, "crashes_dir", String)?; - let inputs_dir = value_t!(args, "inputs_dir", String)?; +pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { + let crashes = value_t!(args, CRASHES_DIR, PathBuf)?.into(); + let inputs = value_t!(args, INPUTS_DIR, PathBuf)?.into(); let target_exe = value_t!(args, "target_exe", PathBuf)?; let target_options = args.values_of_lossy("target_options").unwrap_or_default(); let target_workers = value_t!(args, "target_workers", u64).unwrap_or_default(); - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - + let target_env = get_target_env(args)?; let readonly_inputs = None; - let inputs = SyncedDir { - path: inputs_dir.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/inputs")?)?, - }; - - let crashes = SyncedDir { - path: crashes_dir.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/crashes")?)?, - }; - let ensemble_sync_delay = None; let common = build_common_config(args)?; let config = Config { @@ -52,44 +35,24 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { common, }; + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_fuzz_config(args)?; LibFuzzerFuzzTask::new(config)?.local_run().await } pub fn args(name: &'static str) -> App<'static, 'static> { - SubCommand::with_name(name) - .about("execute a local-only libfuzzer crash report task") - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) - .arg( - Arg::with_name("target_options") - .long("target_options") - .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set target_env first"), - ) - .arg( - Arg::with_name("inputs_dir") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name("crashes_dir") - .takes_value(true) - .required(true), - ) + let mut app = + SubCommand::with_name(name).about("execute a local-only libfuzzer crash report task"); + + app = add_target_cmd_options(true, true, true, app); + app.arg(Arg::with_name(INPUTS_DIR).takes_value(true).required(true)) + .arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) .arg( - Arg::with_name("target_workers") - .long("target_workers") + Arg::with_name(TARGET_WORKERS) + .long(TARGET_WORKERS) .takes_value(true), ) } diff --git a/src/agent/onefuzz-agent/src/local/mod.rs b/src/agent/onefuzz-agent/src/local/mod.rs index 049c4240dd..192649a7a3 100644 --- a/src/agent/onefuzz-agent/src/local/mod.rs +++ b/src/agent/onefuzz-agent/src/local/mod.rs @@ -3,5 +3,6 @@ pub mod cmd; pub mod common; +pub mod libfuzzer; pub mod libfuzzer_crash_report; pub mod libfuzzer_fuzz; diff --git a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs index 603cdc0ed0..888aba9bd3 100644 --- a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs @@ -66,8 +66,9 @@ async fn run_existing(config: &Config) -> Result<()> { async fn already_checked(config: &Config, input: &BlobUrl) -> Result { let result = if let Some(crashes) = &config.crashes { - crashes.url.account() == input.account() - && crashes.url.container() == input.container() + // TODO: this should really check if the URL is None then only check the local path. Otherwise check the container info + crashes.url()?.account() == input.account() + && crashes.url()?.container() == input.container() && crashes.path.join(input.name()).exists() } else { false diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index b01d53b76c..20443950aa 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -146,7 +146,7 @@ impl Config { } Config::LibFuzzerReport(config) => { report::libfuzzer_report::ReportTask::new(config) - .run_managed() + .managed_run() .await } Config::LibFuzzerCoverage(config) => { diff --git a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs index f83f547f78..00f13fa3c2 100644 --- a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs @@ -228,7 +228,7 @@ impl CoverageProcessor { #[async_trait] impl Processor for CoverageProcessor { - async fn process(&mut self, _url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, _url: Option, input: &Path) -> Result<()> { self.heartbeat_client.alive(); self.test_input(input).await?; self.report_total().await?; diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index bb1e48d018..78aaad41d8 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -109,11 +109,14 @@ impl InputPoller { to_process: &SyncedDir, ) -> Result<()> { self.batch_dir = Some(to_process.clone()); - to_process.init_pull().await?; + if to_process.url.is_some() { + to_process.init_pull().await?; + } + info!("checking: {}", to_process.path.display()); let mut read_dir = fs::read_dir(&to_process.path).await?; while let Some(file) = read_dir.next().await { - verbose!("Processing batch-downloaded input {:?}", file); + info!("Processing batch-downloaded input {:?}", file); let file = file?; let path = file.path(); @@ -126,7 +129,7 @@ impl InputPoller { let dir_relative = input_path.strip_prefix(&dir_path)?; dir_relative.display().to_string() }; - let url = to_process.url.blob(blob_name).url(); + let url = to_process.url().map(|x| x.blob(blob_name).url()).ok(); processor.process(url, &path).await?; } @@ -137,8 +140,10 @@ impl InputPoller { pub async fn seen_in_batch(&self, url: &Url) -> Result { let result = if let Some(batch_dir) = &self.batch_dir { if let Ok(blob) = BlobUrl::new(url.clone()) { - batch_dir.url.account() == blob.account() - && batch_dir.url.container() == blob.container() + // TODO - this should see if we have a URL container and only check that part if we do + // otherwise only check the local path + batch_dir.url()?.account() == blob.account() + && batch_dir.url()?.container() == blob.container() && batch_dir.path.join(blob.name()).exists() } else { false @@ -263,7 +268,7 @@ impl InputPoller { } } (Downloaded(msg, url, input), Process(processor)) => { - processor.process(url, &input).await?; + processor.process(Some(url), &input).await?; self.set_state(Processed(msg)); } diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs index 552a04f7ec..8148e0d97c 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs @@ -26,7 +26,7 @@ pub trait Downloader { #[async_trait] pub trait Processor { - async fn process(&mut self, url: Url, input: &Path) -> Result<()>; + async fn process(&mut self, url: Option, input: &Path) -> Result<()>; } pub trait Callback { diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index 4129c56e18..a27a7c4f4c 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -8,18 +8,19 @@ use onefuzz::{ syncdir::SyncedDir, telemetry::Event::{new_report, new_unable_to_reproduce, new_unique_report}, }; - use reqwest::StatusCode; use reqwest_retry::SendRetry; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use tokio::fs; use uuid::Uuid; #[derive(Debug, Deserialize, Serialize)] pub struct CrashReport { pub input_sha256: String, - pub input_blob: InputBlob, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_blob: Option, pub executable: PathBuf, @@ -44,7 +45,8 @@ pub struct CrashReport { #[derive(Debug, Deserialize, Serialize)] pub struct NoCrash { pub input_sha256: String, - pub input_blob: InputBlob, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_blob: Option, pub executable: PathBuf, pub task_id: Uuid, pub job_id: Uuid, @@ -95,6 +97,34 @@ async fn upload_no_repro(report: &NoCrash, container: &BlobContainerUrl) -> Resu } impl CrashTestResult { + pub async fn save_local( + &self, + unique_reports: &SyncedDir, + reports: &Option, + no_repro: &Option, + ) -> Result<()> { + match self { + Self::CrashReport(report) => { + let data = serde_json::to_vec(&report)?; + let unique_path = unique_reports.path.join(report.blob_name()); + fs::write(unique_path, &data).await?; + + if let Some(reports) = reports { + let report_path = reports.path.join(report.blob_name()); + fs::write(report_path, &data).await?; + } + } + Self::NoRepro(report) => { + if let Some(no_repro) = no_repro { + let data = serde_json::to_vec(&report)?; + let no_repro_path = no_repro.path.join(report.blob_name()); + fs::write(no_repro_path, &data).await?; + } + } + } + Ok(()) + } + pub async fn upload( &self, unique_reports: &SyncedDir, @@ -103,14 +133,14 @@ impl CrashTestResult { ) -> Result<()> { match self { Self::CrashReport(report) => { - upload_deduped(report, &unique_reports.url).await?; + upload_deduped(report, unique_reports.url()?).await?; if let Some(reports) = reports { - upload_report(report, &reports.url).await?; + upload_report(report, reports.url()?).await?; } } Self::NoRepro(report) => { if let Some(no_repro) = no_repro { - upload_no_repro(report, &no_repro.url).await?; + upload_no_repro(report, no_repro.url()?).await?; } } } @@ -141,7 +171,7 @@ impl CrashReport { task_id: Uuid, job_id: Uuid, executable: impl Into, - input_blob: InputBlob, + input_blob: Option, input_sha256: String, ) -> Self { Self { diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index d647e1c3eb..40f6573d45 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -106,12 +106,19 @@ impl<'a> GenericReportProcessor<'a> { } } - pub async fn test_input(&self, input_url: Url, input: &Path) -> Result { + pub async fn test_input( + &self, + input_url: Option, + input: &Path, + ) -> Result { self.heartbeat_client.alive(); let input_sha256 = sha256::digest_file(input).await?; let task_id = self.config.common.task_id; let job_id = self.config.common.job_id; - let input_blob = InputBlob::from(BlobUrl::new(input_url)?); + let input_blob = match input_url { + Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)), + None => None, + }; let test_report = self.tester.test_input(input).await?; @@ -161,7 +168,7 @@ impl<'a> GenericReportProcessor<'a> { #[async_trait] impl<'a> Processor for GenericReportProcessor<'a> { - async fn process(&mut self, url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, url: Option, input: &Path) -> Result<()> { let report = self.test_input(url, input).await?; report .upload( diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index ef4f96fc2e..ff107fc116 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -58,30 +58,38 @@ impl ReportTask { Self { config, poller } } - pub async fn run_local(&mut self) -> Result<()> { - let mut processor = AsanProcessor::new(self.config.clone()).await?; + pub async fn local_run(&self) -> Result<()> { + let processor = AsanProcessor::new(self.config.clone()).await?; let crashes = match &self.config.crashes { Some(x) => x, None => bail!("missing crashes directory"), }; - self.poller.batch_process(&mut processor, crashes).await?; + tokio::fs::create_dir_all(&self.config.unique_reports.path).await?; + if let Some(reports) = &self.config.reports { + tokio::fs::create_dir_all(&reports.path).await?; + } + if let Some(no_repro) = &self.config.no_repro { + tokio::fs::create_dir_all(&no_repro.path).await?; + } + + let mut read_dir = tokio::fs::read_dir(&crashes.path).await?; + while let Some(crash) = read_dir.next().await { + processor.test_local(crash?.path()).await?; + } if self.config.check_queue { let mut monitor = DirectoryMonitor::new(crashes.path.clone()); monitor.start()?; - while let Some(crash) = monitor.next().await { - let test_url = Url::parse("https://contoso.com/sample-container/blob.txt")?; - let input_path = Path::new(&crash); - let result = processor.test_input(test_url, &input_path).await?; + processor.test_local(crash).await?; } } Ok(()) } - pub async fn run_managed(&mut self) -> Result<()> { + pub async fn managed_run(&mut self) -> Result<()> { info!("Starting libFuzzer crash report task"); let mut processor = AsanProcessor::new(self.config.clone()).await?; @@ -114,7 +122,25 @@ impl AsanProcessor { }) } - pub async fn test_input(&self, input_url: Url, input: &Path) -> Result { + async fn test_local(&self, input: PathBuf) -> Result<()> { + info!("generating crash report for: {}", input.display()); + let result = self.test_input(None, &input).await?; + result + .save_local( + &self.config.unique_reports, + &self.config.reports, + &self.config.no_repro, + ) + .await?; + + Ok(()) + } + + pub async fn test_input( + &self, + input_url: Option, + input: &Path, + ) -> Result { self.heartbeat_client.alive(); let fuzzer = LibFuzzer::new( &self.config.target_exe, @@ -124,7 +150,10 @@ impl AsanProcessor { let task_id = self.config.common.task_id; let job_id = self.config.common.job_id; - let input_blob = InputBlob::from(BlobUrl::new(input_url)?); + let input_blob = match input_url { + Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)), + None => None, + }; let input_sha256 = sha256::digest_file(input).await?; let test_report = fuzzer @@ -166,7 +195,8 @@ impl AsanProcessor { #[async_trait] impl Processor for AsanProcessor { - async fn process(&mut self, url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, url: Option, input: &Path) -> Result<()> { + info!("processing libfuzzer crash url:{:?} path:{:?}", url, input); let report = self.test_input(url, input).await?; report .upload( diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index e6c308dc46..c87cea1b21 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -26,13 +26,17 @@ const DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS: u64 = 60; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct SyncedDir { pub path: PathBuf, - pub url: BlobContainerUrl, + pub url: Option, } impl SyncedDir { pub async fn sync(&self, operation: SyncOperation, delete_dst: bool) -> Result<()> { + if self.url.is_none() { + bail!("only able to sync with remote URLs"); + } + let dir = &self.path; - let url = self.url.url(); + let url = self.url.as_ref().unwrap().url(); let url = url.as_ref(); verbose!("syncing {:?} {}", operation, dir.display()); match operation { @@ -41,6 +45,14 @@ impl SyncedDir { } } + pub fn url<'a>(&'a self) -> Result<&'a BlobContainerUrl> { + let url = match &self.url { + Some(x) => x, + None => bail!("missing URL context"), + }; + Ok(url) + } + pub async fn init_pull(&self) -> Result<()> { self.init().await?; self.sync(SyncOperation::Pull, false).await @@ -72,6 +84,10 @@ impl SyncedDir { operation: SyncOperation, delay_seconds: Option, ) -> Result<()> { + if self.url.is_none() { + bail!("only able to sync with remote URLs"); + } + let delay_seconds = delay_seconds.unwrap_or(DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS); if delay_seconds == 0 { return Ok(()); @@ -85,7 +101,11 @@ impl SyncedDir { } async fn file_uploader_monitor(&self, event: Event) -> Result<()> { - let url = self.url.url(); + if self.url.is_none() { + bail!("only able to monitor with remote URLs"); + } + + let url = self.url.as_ref().unwrap().url(); verbose!("monitoring {}", self.path.display()); let mut monitor = DirectoryMonitor::new(self.path.clone()); @@ -118,6 +138,10 @@ impl SyncedDir { /// to be initialized, but a user-supplied binary, (such as AFL) logically owns /// a directory, and may reset it. pub async fn monitor_results(&self, event: Event) -> Result<()> { + if self.url.is_none() { + bail!("only able to monitor with remote URLs"); + } + loop { verbose!("waiting to monitor {}", self.path.display()); @@ -132,6 +156,12 @@ impl SyncedDir { } } +impl From for SyncedDir { + fn from(path: PathBuf) -> Self { + Self { path, url: None } + } +} + pub async fn continuous_sync( dirs: &[SyncedDir], operation: SyncOperation, From 5528c0dda43e3f68d71d55289bfdc0c99992cd28 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Sun, 20 Dec 2020 22:57:33 -0500 Subject: [PATCH 03/30] continued dev --- src/agent/onefuzz-agent/src/debug/cmd.rs | 7 +- .../src/debug/libfuzzer_coverage.rs | 96 ------------------- src/agent/onefuzz-agent/src/debug/mod.rs | 1 - src/agent/onefuzz-agent/src/local/cmd.rs | 10 +- .../src/local/libfuzzer_coverage.rs | 64 +++++++++++++ src/agent/onefuzz-agent/src/local/mod.rs | 1 + src/agent/onefuzz-agent/src/tasks/config.rs | 4 +- .../src/tasks/coverage/libfuzzer_coverage.rs | 23 +++-- .../src/tasks/generic/input_poller.rs | 25 +---- .../src/tasks/generic/input_poller/tests.rs | 76 +++++++-------- .../onefuzz-agent/src/tasks/report/generic.rs | 4 +- .../src/tasks/report/libfuzzer_report.rs | 9 +- 12 files changed, 140 insertions(+), 180 deletions(-) delete mode 100644 src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs create mode 100644 src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs diff --git a/src/agent/onefuzz-agent/src/debug/cmd.rs b/src/agent/onefuzz-agent/src/debug/cmd.rs index b4d2eea094..c3bf92bb98 100644 --- a/src/agent/onefuzz-agent/src/debug/cmd.rs +++ b/src/agent/onefuzz-agent/src/debug/cmd.rs @@ -5,18 +5,16 @@ use anyhow::Result; use clap::{App, SubCommand}; use crate::{ - debug::{generic_crash_report, libfuzzer_coverage, libfuzzer_merge}, + debug::{generic_crash_report, libfuzzer_merge}, local::common::add_common_config, }; const GENERIC_CRASH_REPORT: &str = "generic-crash-report"; -const LIBFUZZER_COVERAGE: &str = "libfuzzer-coverage"; const LIBFUZZER_MERGE: &str = "libfuzzer-merge"; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub).await, - (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, (LIBFUZZER_MERGE, Some(sub)) => libfuzzer_merge::run(sub).await, _ => { anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); @@ -30,8 +28,5 @@ pub fn args(name: &str) -> App<'static, 'static> { .subcommand(add_common_config(generic_crash_report::args( GENERIC_CRASH_REPORT, ))) - .subcommand(add_common_config(libfuzzer_coverage::args( - LIBFUZZER_COVERAGE, - ))) .subcommand(add_common_config(libfuzzer_merge::args(LIBFUZZER_MERGE))) } diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs deleted file mode 100644 index a8ed2ce2e2..0000000000 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use crate::{ - local::common::build_common_config, - tasks::{ - coverage::libfuzzer_coverage::{Config, CoverageProcessor}, - utils::parse_key_value, - }, -}; -use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use onefuzz::syncdir::SyncedDir; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; - -async fn run_impl(input: String, config: Config) -> Result<()> { - let mut processor = CoverageProcessor::new(Arc::new(config)) - .await - .map_err(|e| format_err!("coverage processor failed: {:?}", e))?; - let input_path = Path::new(&input); - processor - .test_input(input_path) - .await - .map_err(|e| format_err!("test input failed {:?}", e))?; - let info = processor - .total - .info() - .await - .map_err(|e| format_err!("coverage_info failed {:?}", e))?; - println!("{:?}", info); - Ok(()) -} - -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let input = value_t!(args, "input", String)?; - let result_dir = value_t!(args, "result_dir", String)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - - let common = build_common_config(args)?; - let config = Config { - target_exe, - target_env, - target_options, - input_queue: None, - readonly_inputs: vec![], - coverage: SyncedDir { - path: result_dir.into(), - url: None, - }, - common, - }; - - run_impl(input, config).await -} - -pub fn args(name: &'static str) -> App<'static, 'static> { - SubCommand::with_name(name) - .about("execute a local-only libfuzzer coverage task") - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name("input").takes_value(true).required(true)) - .arg( - Arg::with_name("result_dir") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) - .arg( - Arg::with_name("target_options") - .long("target_options") - .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .default_value("{input}") - .help("Supports hyphens. Recommendation: Set target_env first"), - ) -} diff --git a/src/agent/onefuzz-agent/src/debug/mod.rs b/src/agent/onefuzz-agent/src/debug/mod.rs index be0d0e0016..d357dbd778 100644 --- a/src/agent/onefuzz-agent/src/debug/mod.rs +++ b/src/agent/onefuzz-agent/src/debug/mod.rs @@ -3,5 +3,4 @@ pub mod cmd; pub mod generic_crash_report; -pub mod libfuzzer_coverage; pub mod libfuzzer_merge; diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs index 5b7c3b31f9..929d17d06f 100644 --- a/src/agent/onefuzz-agent/src/local/cmd.rs +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -4,16 +4,21 @@ use anyhow::Result; use clap::{App, SubCommand}; -use crate::local::{common::add_common_config, libfuzzer, libfuzzer_crash_report, libfuzzer_fuzz}; +use crate::local::{ + common::add_common_config, libfuzzer, libfuzzer_coverage, libfuzzer_crash_report, + libfuzzer_fuzz, +}; const LIBFUZZER: &str = "libfuzzer"; const LIBFUZZER_FUZZ: &str = "libfuzzer-fuzz"; const LIBFUZZER_CRASH_REPORT: &str = "libfuzzer-crash-report"; +const LIBFUZZER_COVERAGE: &str = "libfuzzer-coverage"; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { (LIBFUZZER, Some(sub)) => libfuzzer::run(sub).await, (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, + (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, _ => { anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); @@ -26,6 +31,9 @@ pub fn args(name: &str) -> App<'static, 'static> { .about("pre-release local fuzzing") .subcommand(add_common_config(libfuzzer::args(LIBFUZZER))) .subcommand(add_common_config(libfuzzer_fuzz::args(LIBFUZZER_FUZZ))) + .subcommand(add_common_config(libfuzzer_coverage::args( + LIBFUZZER_COVERAGE, + ))) .subcommand(add_common_config(libfuzzer_crash_report::args( LIBFUZZER_CRASH_REPORT, ))) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs new file mode 100644 index 0000000000..9edb9e9237 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + add_target_cmd_options, build_common_config, get_target_env, TARGET_EXE, TARGET_OPTIONS, + }, + tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +const COVERAGE_DIR: &str = "coverage_dir"; +const READONLY_INPUTS: &str = "readonly_inputs_dir"; + +pub fn build_coverage_config(args: &clap::ArgMatches<'_>) -> Result { + let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; + let readonly_inputs = values_t!(args, READONLY_INPUTS, PathBuf)? + .iter() + .map(|x| x.to_owned().into()) + .collect(); + let coverage = value_t!(args, COVERAGE_DIR, PathBuf)?.into(); + let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); + + let target_env = get_target_env(args)?; + + let common = build_common_config(args)?; + let config = Config { + target_exe, + target_env, + target_options, + input_queue: None, + readonly_inputs, + coverage, + common, + }; + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_coverage_config(args)?; + + let task = CoverageTask::new(config); + task.local_run().await +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer coverage task"); + + app = add_target_cmd_options(true, true, true, app); + + app.arg( + Arg::with_name(COVERAGE_DIR) + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name(READONLY_INPUTS) + .takes_value(true) + .required(true) + .multiple(true), + ) +} diff --git a/src/agent/onefuzz-agent/src/local/mod.rs b/src/agent/onefuzz-agent/src/local/mod.rs index 192649a7a3..8bca72aad4 100644 --- a/src/agent/onefuzz-agent/src/local/mod.rs +++ b/src/agent/onefuzz-agent/src/local/mod.rs @@ -4,5 +4,6 @@ pub mod cmd; pub mod common; pub mod libfuzzer; +pub mod libfuzzer_coverage; pub mod libfuzzer_crash_report; pub mod libfuzzer_fuzz; diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 20443950aa..0325e5a507 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -150,8 +150,8 @@ impl Config { .await } Config::LibFuzzerCoverage(config) => { - coverage::libfuzzer_coverage::CoverageTask::new(Arc::new(config)) - .run() + coverage::libfuzzer_coverage::CoverageTask::new(config) + .managed_run() .await } Config::LibFuzzerMerge(config) => merge::libfuzzer_merge::spawn(Arc::new(config)).await, diff --git a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs index 00f13fa3c2..3712793d06 100644 --- a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs @@ -79,17 +79,26 @@ pub struct CoverageTask { } impl CoverageTask { - pub fn new(config: impl Into>) -> Self { - let config = config.into(); + pub fn new(config: Config) -> Self { + let config = Arc::new(config); + let poller = InputPoller::new(); + Self { config, poller } + } - let task_dir = PathBuf::from(config.common.task_id.to_string()); - let poller_dir = task_dir.join("poller"); - let poller = InputPoller::::new(poller_dir); + pub async fn local_run(&self) -> Result<()> { + let mut processor = CoverageProcessor::new(self.config.clone()).await?; - Self { config, poller } + self.config.coverage.init().await?; + for synced_dir in &self.config.readonly_inputs { + synced_dir.init().await?; + self.record_corpus_coverage(&mut processor, &synced_dir) + .await?; + } + + Ok(()) } - pub async fn run(&mut self) -> Result<()> { + pub async fn managed_run(&mut self) -> Result<()> { info!("starting libFuzzer coverage task"); self.config.coverage.init_pull().await?; self.process().await diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index 78aaad41d8..ddab7d1d42 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -5,8 +5,9 @@ use std::{fmt, path::PathBuf}; use anyhow::Result; use futures::stream::StreamExt; -use onefuzz::{blob::BlobUrl, fs::OwnedDir, jitter::delay_with_jitter, syncdir::SyncedDir}; +use onefuzz::{blob::BlobUrl, jitter::delay_with_jitter, syncdir::SyncedDir}; use reqwest::Url; +use tempfile::tempdir; use tokio::{fs, time::Duration}; mod callback; @@ -78,10 +79,6 @@ impl<'a, M> fmt::Debug for Event<'a, M> { /// application data (here, the input URL, in some encoding) and metadata for /// operations like finalizing a dequeue with a pop receipt. pub struct InputPoller { - /// Agent-local directory where the poller will download inputs. - /// Will be reset for each new input. - working_dir: OwnedDir, - /// Internal automaton state. /// /// This is only nullable so we can internally `take()` the current state @@ -92,12 +89,10 @@ pub struct InputPoller { } impl InputPoller { - pub fn new(working_dir: impl Into) -> Self { - let working_dir = OwnedDir::new(working_dir); + pub fn new() -> Self { let state = Some(State::Ready); Self { state, - working_dir, batch_dir: None, } } @@ -154,15 +149,6 @@ impl InputPoller { Ok(result) } - /// Path to the working directory. - /// - /// We will create or reset the working directory before entering the - /// `Downloaded` state, but a caller cannot otherwise assume it exists. - #[allow(unused)] - pub fn working_dir(&self) -> &OwnedDir { - &self.working_dir - } - /// Get the current automaton state, including the state data. pub fn state(&self) -> &State { self.state.as_ref().unwrap_or_else(|| unreachable!()) @@ -254,14 +240,13 @@ impl InputPoller { } } (Parsed(msg, url), Download(downloader)) => { - self.working_dir.reset().await?; - + let download_dir = tempdir()?; if self.seen_in_batch(&url).await? { verbose!("url was seen during batch processing: {:?}", url); self.set_state(Processed(msg)); } else { let input = downloader - .download(url.clone(), self.working_dir.path()) + .download(url.clone(), download_dir.path()) .await?; self.set_state(Downloaded(msg, url, input)); diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs index 342b0a698a..f770ef24f4 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs @@ -5,7 +5,6 @@ use anyhow::Result; use async_trait::async_trait; use reqwest::Url; use std::path::Path; -use tempfile::{tempdir, TempDir}; use super::*; @@ -84,12 +83,12 @@ impl Downloader for TestDownloader { #[derive(Default)] struct TestProcessor { - processed: Vec<(Url, PathBuf)>, + processed: Vec<(Option, PathBuf)>, } #[async_trait] impl Processor for TestProcessor { - async fn process(&mut self, url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, url: Option, input: &Path) -> Result<()> { self.processed.push((url, input.to_owned())); Ok(()) @@ -100,11 +99,10 @@ fn url_input_name(url: &Url) -> String { url.path_segments().unwrap().last().unwrap().to_owned() } -fn fixture() -> (TempDir, InputPoller) { - let dir = tempdir().unwrap(); - let task = InputPoller::new(dir.path()); +fn fixture() -> InputPoller { + let task = InputPoller::new(); - (dir, task) + task } fn url_fixture(msg: Msg) -> Url { @@ -118,7 +116,7 @@ fn input_fixture(dir: &Path, msg: Msg) -> PathBuf { #[tokio::test] async fn test_ready_poll() { - let (_, mut task) = fixture(); + let mut task = fixture(); let msg: Msg = 0; @@ -133,7 +131,7 @@ async fn test_ready_poll() { #[tokio::test] async fn test_polled_some_parse() { - let (_, mut task) = fixture(); + let mut task = fixture(); let msg: Msg = 0; let url = url_fixture(msg); @@ -150,7 +148,7 @@ async fn test_polled_some_parse() { #[tokio::test] async fn test_polled_none_parse() { - let (_, mut task) = fixture(); + let mut task = fixture(); task.set_state(State::Polled(None)); @@ -162,47 +160,47 @@ async fn test_polled_none_parse() { assert_eq!(task.state(), &State::Ready); } -#[tokio::test] -async fn test_parsed_download() { - let (dir, mut task) = fixture(); +// #[tokio::test] +// async fn test_parsed_download() { +// let mut task = fixture(); - let msg: Msg = 0; - let url = url_fixture(msg); - let input = input_fixture(dir.path(), msg); +// let msg: Msg = 0; +// let url = url_fixture(msg); +// let input = input_fixture(dir.path(), msg); - task.set_state(State::Parsed(msg, url.clone())); +// task.set_state(State::Parsed(msg, url.clone())); - let mut downloader = TestDownloader::default(); +// let mut downloader = TestDownloader::default(); - task.trigger(Event::Download(&mut downloader)) - .await - .unwrap(); +// task.trigger(Event::Download(&mut downloader)) +// .await +// .unwrap(); - assert_eq!(task.state(), &State::Downloaded(msg, url.clone(), input)); - assert_eq!(downloader.downloaded, vec![url]); -} +// assert_eq!(task.state(), &State::Downloaded(msg, url.clone(), input)); +// assert_eq!(downloader.downloaded, vec![url]); +// } -#[tokio::test] -async fn test_downloaded_process() { - let (dir, mut task) = fixture(); +// #[tokio::test] +// async fn test_downloaded_process() { +// let mut task = fixture(); - let msg: Msg = 0; - let url = url_fixture(msg); - let input = input_fixture(dir.path(), msg); +// let msg: Msg = 0; +// let url = url_fixture(msg); +// let input = input_fixture(dir.path(), msg); - task.set_state(State::Downloaded(msg, url.clone(), input.clone())); +// task.set_state(State::Downloaded(msg, url.clone(), input.clone())); - let mut processor = TestProcessor::default(); +// let mut processor = TestProcessor::default(); - task.trigger(Event::Process(&mut processor)).await.unwrap(); +// task.trigger(Event::Process(&mut processor)).await.unwrap(); - assert_eq!(task.state(), &State::Processed(msg)); - assert_eq!(processor.processed, vec![(url, input)]); -} +// assert_eq!(task.state(), &State::Processed(msg)); +// assert_eq!(processor.processed, vec![(Some(url), input)]); +// } #[tokio::test] async fn test_processed_finish() { - let (_, mut task) = fixture(); + let mut task = fixture(); let msg: Msg = 0; @@ -218,7 +216,7 @@ async fn test_processed_finish() { #[tokio::test] async fn test_invalid_trigger() { - let (_, mut task) = fixture(); + let mut task = fixture(); let mut queue = TestQueue::default(); @@ -231,7 +229,7 @@ async fn test_invalid_trigger() { #[tokio::test] async fn test_valid_trigger_failed_action() { - let (_, mut task) = fixture(); + let mut task = fixture(); let mut queue = TestQueueAlwaysFails; diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index 40f6573d45..6f6938e070 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -57,9 +57,7 @@ pub struct ReportTask<'a> { impl<'a> ReportTask<'a> { pub fn new(config: &'a Config) -> Self { - let working_dir = config.common.task_id.to_string(); - let poller = InputPoller::new(working_dir); - + let poller = InputPoller::new(); Self { config, poller } } diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index ff107fc116..e43da506a4 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -51,8 +51,7 @@ pub struct ReportTask { impl ReportTask { pub fn new(config: Config) -> Self { - let working_dir = config.common.task_id.to_string(); - let poller = InputPoller::new(working_dir); + let poller = InputPoller::new(); let config = Arc::new(config); Self { config, poller } @@ -65,12 +64,12 @@ impl ReportTask { None => bail!("missing crashes directory"), }; - tokio::fs::create_dir_all(&self.config.unique_reports.path).await?; + self.config.unique_reports.init().await?; if let Some(reports) = &self.config.reports { - tokio::fs::create_dir_all(&reports.path).await?; + reports.init().await?; } if let Some(no_repro) = &self.config.no_repro { - tokio::fs::create_dir_all(&no_repro.path).await?; + no_repro.init().await?; } let mut read_dir = tokio::fs::read_dir(&crashes.path).await?; From c23109c79a3f8c2a875fe616055bf107aa6baa4d Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Sun, 20 Dec 2020 23:44:41 -0500 Subject: [PATCH 04/30] optionally enable coverage as part of the libfuzzer local fuzzing --- src/agent/onefuzz-agent/src/local/common.rs | 1 + .../onefuzz-agent/src/local/libfuzzer.rs | 27 +++++++++++++++---- .../src/local/libfuzzer_coverage.rs | 22 +++++++++------ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 1036ff8503..35b121b63f 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -17,6 +17,7 @@ pub const TARGET_TIMEOUT: &str = "target_timeout"; pub const CHECK_RETRY_COUNT: &str = "check_retry_count"; pub const DISABLE_CHECK_QUEUE: &str = "disable_check_queue"; pub const UNIQUE_REPORTS_DIR: &str = "unique_reports_dir"; +pub const COVERAGE_DIR: &str = "coverage_dir"; pub fn add_target_cmd_options( exe: bool, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index 871f78b0ee..97e08b4bc5 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -4,14 +4,18 @@ use crate::{ local::{ common::{ - add_target_cmd_options, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, - INPUTS_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, - UNIQUE_REPORTS_DIR, + add_target_cmd_options, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, + DISABLE_CHECK_QUEUE, INPUTS_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, + TARGET_WORKERS, UNIQUE_REPORTS_DIR, }, + libfuzzer_coverage::build_coverage_config, libfuzzer_crash_report::build_report_config, libfuzzer_fuzz::build_fuzz_config, }, - tasks::{fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask, report::libfuzzer_report::ReportTask}, + tasks::{ + coverage::libfuzzer_coverage::CoverageTask, fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask, + report::libfuzzer_report::ReportTask, + }, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; @@ -25,8 +29,15 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let report = ReportTask::new(report_config); let report_task = report.local_run(); + if args.is_present(COVERAGE_DIR) { + let coverage_config = build_coverage_config(args, true)?; + let coverage = CoverageTask::new(coverage_config); + let coverage_task = coverage.local_run(); - tokio::try_join!(fuzz_task, report_task)?; + tokio::try_join!(fuzz_task, report_task, coverage_task)?; + } else { + tokio::try_join!(fuzz_task, report_task)?; + } Ok(()) } @@ -49,6 +60,12 @@ pub fn args(name: &'static str) -> App<'static, 'static> { .takes_value(true) .required(false), ) + .arg( + Arg::with_name(COVERAGE_DIR) + .long(COVERAGE_DIR) + .takes_value(true) + .required(false), + ) .arg( Arg::with_name(NO_REPRO_DIR) .long(NO_REPRO_DIR) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 9edb9e9237..87a276a9d8 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -3,7 +3,8 @@ use crate::{ local::common::{ - add_target_cmd_options, build_common_config, get_target_env, TARGET_EXE, TARGET_OPTIONS, + add_target_cmd_options, build_common_config, get_target_env, COVERAGE_DIR, INPUTS_DIR, + TARGET_EXE, TARGET_OPTIONS, }, tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, }; @@ -11,15 +12,20 @@ use anyhow::Result; use clap::{App, Arg, SubCommand}; use std::path::PathBuf; -const COVERAGE_DIR: &str = "coverage_dir"; const READONLY_INPUTS: &str = "readonly_inputs_dir"; -pub fn build_coverage_config(args: &clap::ArgMatches<'_>) -> Result { +pub fn build_coverage_config(args: &clap::ArgMatches<'_>, use_inputs: bool) -> Result { let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; - let readonly_inputs = values_t!(args, READONLY_INPUTS, PathBuf)? - .iter() - .map(|x| x.to_owned().into()) - .collect(); + + let readonly_inputs = if use_inputs { + vec![value_t!(args, INPUTS_DIR, PathBuf)?.into()] + } else { + values_t!(args, READONLY_INPUTS, PathBuf)? + .iter() + .map(|x| x.to_owned().into()) + .collect() + }; + let coverage = value_t!(args, COVERAGE_DIR, PathBuf)?.into(); let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); @@ -39,7 +45,7 @@ pub fn build_coverage_config(args: &clap::ArgMatches<'_>) -> Result { } pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let config = build_coverage_config(args)?; + let config = build_coverage_config(args, false)?; let task = CoverageTask::new(config); task.local_run().await From 09da0707d564a68c87d310f84dc3514353027355 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 22 Dec 2020 15:33:21 -0500 Subject: [PATCH 05/30] make the managed mode more clear --- src/agent/onefuzz-agent/src/main.rs | 60 +++++----------------- src/agent/onefuzz-agent/src/managed/cmd.rs | 33 ++++++++++++ src/agent/onefuzz-agent/src/managed/mod.rs | 1 + src/agent/onefuzz-supervisor/src/worker.rs | 2 +- 4 files changed, 48 insertions(+), 48 deletions(-) create mode 100644 src/agent/onefuzz-agent/src/managed/cmd.rs create mode 100644 src/agent/onefuzz-agent/src/managed/mod.rs diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 455f8641cb..0ad9c284fe 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -10,21 +10,19 @@ extern crate onefuzz; #[macro_use] extern crate clap; -use std::path::PathBuf; - use anyhow::Result; -use clap::{App, Arg, ArgMatches, SubCommand}; -use onefuzz::telemetry::{self}; +use clap::{App, ArgMatches, SubCommand}; use std::io::{stdout, Write}; mod debug; mod local; +mod managed; mod tasks; -use tasks::config::{CommonConfig, Config}; - +const LICENSE_CMD: &str = "licenses"; const LOCAL_CMD: &str = "local"; const DEBUG_CMD: &str = "debug"; +const MANAGED_CMD: &str = "managed"; fn main() -> Result<()> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); @@ -38,15 +36,10 @@ fn main() -> Result<()> { let app = App::new("onefuzz-agent") .version(built_version.as_str()) - .arg( - Arg::with_name("config") - .long("config") - .short("c") - .takes_value(true), - ) + .subcommand(managed::cmd::args(MANAGED_CMD)) .subcommand(local::cmd::args(LOCAL_CMD)) .subcommand(debug::cmd::args(DEBUG_CMD)) - .subcommand(SubCommand::with_name("licenses").about("display third-party licenses")); + .subcommand(SubCommand::with_name(LICENSE_CMD).about("display third-party licenses")); let matches = app.get_matches(); @@ -54,46 +47,19 @@ fn main() -> Result<()> { rt.block_on(run(matches)) } -async fn run(matches: ArgMatches<'_>) -> Result<()> { - match matches.subcommand() { - ("licenses", Some(_)) => { - return licenses(); - } +async fn run(args: ArgMatches<'_>) -> Result<()> { + match args.subcommand() { + (LICENSE_CMD, Some(_)) => return licenses(), (DEBUG_CMD, Some(sub)) => return debug::cmd::run(sub).await, (LOCAL_CMD, Some(sub)) => return local::cmd::run(sub).await, - _ => {} // no subcommand - } - - if matches.value_of("config").is_none() { - println!("Missing '--config'\n{}", matches.usage()); - return Ok(()); - } - - let config_arg = matches.value_of("config").unwrap(); - run_config(config_arg).await -} - -async fn run_config(config_arg: &str) -> Result<()> { - let config_path: PathBuf = config_arg.parse()?; - let config = Config::from_file(config_path)?; - - init_telemetry(config.common()); - verbose!("config parsed"); - let result = config.run().await; - - if let Err(err) = &result { - error!("error running task: {}", err); + (MANAGED_CMD, Some(sub)) => return managed::cmd::run(sub).await, + _ => { + anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); + } } - - telemetry::try_flush_and_close(); - result } fn licenses() -> Result<()> { stdout().write_all(include_bytes!("../../data/licenses.json"))?; Ok(()) } - -fn init_telemetry(config: &CommonConfig) { - telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key); -} diff --git a/src/agent/onefuzz-agent/src/managed/cmd.rs b/src/agent/onefuzz-agent/src/managed/cmd.rs new file mode 100644 index 0000000000..89c0376151 --- /dev/null +++ b/src/agent/onefuzz-agent/src/managed/cmd.rs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::tasks::config::{CommonConfig, Config}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use onefuzz::telemetry; +use std::path::PathBuf; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config_path = value_t!(args, "config", PathBuf)?; + let config = Config::from_file(config_path)?; + + init_telemetry(config.common()); + let result = config.run().await; + + if let Err(err) = &result { + error!("error running task: {}", err); + } + + telemetry::try_flush_and_close(); + result +} + +fn init_telemetry(config: &CommonConfig) { + telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key); +} + +pub fn args(name: &str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("managed fuzzing") + .arg(Arg::with_name("config").required(true)) +} diff --git a/src/agent/onefuzz-agent/src/managed/mod.rs b/src/agent/onefuzz-agent/src/managed/mod.rs new file mode 100644 index 0000000000..52958ec91f --- /dev/null +++ b/src/agent/onefuzz-agent/src/managed/mod.rs @@ -0,0 +1 @@ +pub mod cmd; diff --git a/src/agent/onefuzz-supervisor/src/worker.rs b/src/agent/onefuzz-supervisor/src/worker.rs index 0853811778..cd4cc26ff0 100644 --- a/src/agent/onefuzz-supervisor/src/worker.rs +++ b/src/agent/onefuzz-supervisor/src/worker.rs @@ -215,7 +215,7 @@ impl IWorkerRunner for WorkerRunner { let mut cmd = Command::new("onefuzz-agent"); cmd.current_dir(&working_dir); - cmd.arg("-c"); + cmd.arg("managed"); cmd.arg("config.json"); cmd.stderr(Stdio::piped()); cmd.stdout(Stdio::piped()); From 2352677f0856270e68296cbee60590b2de4c3ef9 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 22 Dec 2020 16:09:14 -0500 Subject: [PATCH 06/30] expose generic crash reporting task --- src/agent/onefuzz-agent/src/debug/cmd.rs | 10 +- .../src/debug/generic_crash_report.rs | 115 ---------------- src/agent/onefuzz-agent/src/debug/mod.rs | 1 - src/agent/onefuzz-agent/src/local/cmd.rs | 9 +- .../src/local/generic_crash_report.rs | 125 ++++++++++++++++++ .../src/local/libfuzzer_coverage.rs | 1 + .../src/local/libfuzzer_crash_report.rs | 2 +- src/agent/onefuzz-agent/src/local/mod.rs | 1 + src/agent/onefuzz-agent/src/tasks/config.rs | 6 +- .../src/tasks/coverage/libfuzzer_coverage.rs | 7 + .../onefuzz-agent/src/tasks/report/generic.rs | 56 +++++++- 11 files changed, 199 insertions(+), 134 deletions(-) delete mode 100644 src/agent/onefuzz-agent/src/debug/generic_crash_report.rs create mode 100644 src/agent/onefuzz-agent/src/local/generic_crash_report.rs diff --git a/src/agent/onefuzz-agent/src/debug/cmd.rs b/src/agent/onefuzz-agent/src/debug/cmd.rs index c3bf92bb98..15cf6d0ab4 100644 --- a/src/agent/onefuzz-agent/src/debug/cmd.rs +++ b/src/agent/onefuzz-agent/src/debug/cmd.rs @@ -4,17 +4,12 @@ use anyhow::Result; use clap::{App, SubCommand}; -use crate::{ - debug::{generic_crash_report, libfuzzer_merge}, - local::common::add_common_config, -}; +use crate::{debug::libfuzzer_merge, local::common::add_common_config}; -const GENERIC_CRASH_REPORT: &str = "generic-crash-report"; const LIBFUZZER_MERGE: &str = "libfuzzer-merge"; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { - (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub).await, (LIBFUZZER_MERGE, Some(sub)) => libfuzzer_merge::run(sub).await, _ => { anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); @@ -25,8 +20,5 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub fn args(name: &str) -> App<'static, 'static> { SubCommand::with_name(name) .about("unsupported internal debugging commands") - .subcommand(add_common_config(generic_crash_report::args( - GENERIC_CRASH_REPORT, - ))) .subcommand(add_common_config(libfuzzer_merge::args(LIBFUZZER_MERGE))) } diff --git a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs b/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs deleted file mode 100644 index 68a391e208..0000000000 --- a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use crate::{ - local::common::build_common_config, - tasks::{ - report::generic::{Config, GenericReportProcessor}, - utils::parse_key_value, - }, -}; -use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use onefuzz::syncdir::SyncedDir; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; -use url::Url; - -async fn run_impl(input: String, config: Config) -> Result<()> { - let input_path = Path::new(&input); - let test_url = Url::parse("https://contoso.com/sample-container/blob.txt")?; - let heartbeat_client = config.common.init_heartbeat().await?; - let processor = GenericReportProcessor::new(&config, heartbeat_client); - let result = processor.test_input(Some(test_url), input_path).await?; - println!("{:#?}", result); - Ok(()) -} - -pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let input = value_t!(args, "input", String)?; - let target_timeout = value_t!(args, "target_timeout", u64).ok(); - let check_retry_count = value_t!(args, "check_retry_count", u64)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - let check_asan_log = args.is_present("check_asan_log"); - let check_debugger = !args.is_present("disable_check_debugger"); - - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - - let common = build_common_config(args)?; - - let config = Config { - target_exe, - target_env, - target_options, - target_timeout, - check_asan_log, - check_debugger, - check_retry_count, - crashes: None, - input_queue: None, - no_repro: None, - reports: None, - unique_reports: SyncedDir { - path: "unique_reports".into(), - url: None, - }, - common, - }; - - run_impl(input, config).await -} - -pub fn args(name: &'static str) -> App<'static, 'static> { - SubCommand::with_name(name) - .about("execute a local-only generic crash report") - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name("input").takes_value(true).required(true)) - .arg( - Arg::with_name("disable_check_debugger") - .takes_value(false) - .long("disable_check_debugger"), - ) - .arg( - Arg::with_name("check_asan_log") - .takes_value(false) - .long("check_asan_log"), - ) - .arg( - Arg::with_name("check_retry_count") - .takes_value(true) - .long("check_retry_count") - .default_value("0"), - ) - .arg( - Arg::with_name("target_timeout") - .takes_value(true) - .long("target_timeout") - .default_value("5"), - ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) - .arg( - Arg::with_name("target_options") - .long("target_options") - .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .default_value("{input}") - .help("Supports hyphens. Recommendation: Set target_env first"), - ) -} diff --git a/src/agent/onefuzz-agent/src/debug/mod.rs b/src/agent/onefuzz-agent/src/debug/mod.rs index d357dbd778..0a7fefd34b 100644 --- a/src/agent/onefuzz-agent/src/debug/mod.rs +++ b/src/agent/onefuzz-agent/src/debug/mod.rs @@ -2,5 +2,4 @@ // Licensed under the MIT License. pub mod cmd; -pub mod generic_crash_report; pub mod libfuzzer_merge; diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs index 929d17d06f..85ba1e519c 100644 --- a/src/agent/onefuzz-agent/src/local/cmd.rs +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -5,14 +5,15 @@ use anyhow::Result; use clap::{App, SubCommand}; use crate::local::{ - common::add_common_config, libfuzzer, libfuzzer_coverage, libfuzzer_crash_report, - libfuzzer_fuzz, + common::add_common_config, generic_crash_report, libfuzzer, libfuzzer_coverage, + libfuzzer_crash_report, libfuzzer_fuzz, }; const LIBFUZZER: &str = "libfuzzer"; const LIBFUZZER_FUZZ: &str = "libfuzzer-fuzz"; const LIBFUZZER_CRASH_REPORT: &str = "libfuzzer-crash-report"; const LIBFUZZER_COVERAGE: &str = "libfuzzer-coverage"; +const GENERIC_CRASH_REPORT: &str = "generic-crash-report"; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { @@ -20,6 +21,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, + (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub).await, _ => { anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); } @@ -37,4 +39,7 @@ pub fn args(name: &str) -> App<'static, 'static> { .subcommand(add_common_config(libfuzzer_crash_report::args( LIBFUZZER_CRASH_REPORT, ))) + .subcommand(add_common_config(generic_crash_report::args( + GENERIC_CRASH_REPORT, + ))) } diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs new file mode 100644 index 0000000000..8627a2afcd --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + add_target_cmd_options, build_common_config, get_target_env, CHECK_RETRY_COUNT, + CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_EXE, TARGET_OPTIONS, + TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + }, + tasks::report::generic::{Config, ReportTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +pub async fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { + let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; + let crashes = Some(value_t!(args, CRASHES_DIR, PathBuf)?.into()); + let reports = if args.is_present(REPORTS_DIR) { + Some(value_t!(args, REPORTS_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + let no_repro = if args.is_present(NO_REPRO_DIR) { + Some(value_t!(args, NO_REPRO_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); + + let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); + let target_env = get_target_env(args)?; + + let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); + + let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; + let check_queue = !args.is_present(DISABLE_CHECK_QUEUE); + let check_asan_log = args.is_present("check_asan_log"); + let check_debugger = !args.is_present("disable_check_debugger"); + + let common = build_common_config(args)?; + + let config = Config { + target_exe, + target_env, + target_options, + target_timeout, + check_asan_log, + check_debugger, + check_retry_count, + check_queue, + crashes, + input_queue: None, + no_repro, + reports, + unique_reports, + common, + }; + + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_report_config(args).await?; + ReportTask::new(&config).local_run().await +} + +pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { + app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) + .arg( + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name(UNIQUE_REPORTS_DIR) + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT) + .default_value("5"), + ) + .arg( + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + ) + .arg( + Arg::with_name(DISABLE_CHECK_QUEUE) + .takes_value(false) + .long(DISABLE_CHECK_QUEUE), + ) + .arg( + Arg::with_name("check_asan_log") + .takes_value(false) + .long("check_asan_log"), + ) + .arg( + Arg::with_name("disable_check_debugger") + .takes_value(false) + .long("disable_check_debugger"), + ) + .arg( + Arg::with_name("disable_check_queue") + .takes_value(false) + .long("disable_check_queue"), + ) +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("execute a local-only generic crash report"); + app = add_target_cmd_options(true, true, true, app); + add_report_options(app) +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 87a276a9d8..5ff6d4a8eb 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -40,6 +40,7 @@ pub fn build_coverage_config(args: &clap::ArgMatches<'_>, use_inputs: bool) -> R readonly_inputs, coverage, common, + check_queue: false, }; Ok(config) } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index ee87459d2e..a603f0f506 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -26,13 +26,13 @@ pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { } else { None }; - let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); let target_env = get_target_env(args)?; let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); + let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; let check_queue = !args.is_present(DISABLE_CHECK_QUEUE); diff --git a/src/agent/onefuzz-agent/src/local/mod.rs b/src/agent/onefuzz-agent/src/local/mod.rs index 8bca72aad4..953ff124eb 100644 --- a/src/agent/onefuzz-agent/src/local/mod.rs +++ b/src/agent/onefuzz-agent/src/local/mod.rs @@ -3,6 +3,7 @@ pub mod cmd; pub mod common; +pub mod generic_crash_report; pub mod libfuzzer; pub mod libfuzzer_coverage; pub mod libfuzzer_crash_report; diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 0325e5a507..3e0fa25077 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -159,7 +159,11 @@ impl Config { Config::GenericGenerator(config) => fuzz::generator::spawn(Arc::new(config)).await, Config::GenericSupervisor(config) => fuzz::supervisor::spawn(config).await, Config::GenericMerge(config) => merge::generic::spawn(Arc::new(config)).await, - Config::GenericReport(config) => report::generic::ReportTask::new(&config).run().await, + Config::GenericReport(config) => { + report::generic::ReportTask::new(&config) + .managed_run() + .await + } } } } diff --git a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs index 3712793d06..8e0893fdd3 100644 --- a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs @@ -52,6 +52,10 @@ use tokio::fs; const TOTAL_COVERAGE: &str = "total.cov"; +fn default_bool_true() -> bool { + true +} + #[derive(Debug, Deserialize)] pub struct Config { pub target_exe: PathBuf, @@ -61,6 +65,9 @@ pub struct Config { pub readonly_inputs: Vec, pub coverage: SyncedDir, + #[serde(default = "default_bool_true")] + pub check_queue: bool, + #[serde(flatten)] pub common: CommonConfig, } diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index 6f6938e070..bcf4f4364c 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -9,7 +9,10 @@ use crate::tasks::{ }; use anyhow::Result; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, input_tester::Tester, sha256, syncdir::SyncedDir}; +use futures::stream::StreamExt; +use onefuzz::{ + blob::BlobUrl, input_tester::Tester, monitor::DirectoryMonitor, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -46,6 +49,9 @@ pub struct Config { #[serde(default)] pub check_retry_count: u64, + #[serde(default = "default_bool_true")] + pub check_queue: bool, + #[serde(flatten)] pub common: CommonConfig, } @@ -61,7 +67,31 @@ impl<'a> ReportTask<'a> { Self { config, poller } } - pub async fn run(&mut self) -> Result<()> { + pub async fn local_run(&mut self) -> Result<()> { + let processor = GenericReportProcessor::new(&self.config, None); + + info!("Starting generic crash report task"); + let crashes = match &self.config.crashes { + Some(x) => x, + None => bail!("missing crashes directory"), + }; + + let mut read_dir = tokio::fs::read_dir(&crashes.path).await?; + while let Some(crash) = read_dir.next().await { + processor.test_local(crash?.path()).await?; + } + + if self.config.check_queue { + let mut monitor = DirectoryMonitor::new(&crashes.path); + monitor.start()?; + while let Some(crash) = monitor.next().await { + processor.test_local(crash).await?; + } + } + Ok(()) + } + + pub async fn managed_run(&mut self) -> Result<()> { info!("Starting generic crash report task"); let heartbeat_client = self.config.common.init_heartbeat().await?; let mut processor = GenericReportProcessor::new(&self.config, heartbeat_client); @@ -70,9 +100,11 @@ impl<'a> ReportTask<'a> { self.poller.batch_process(&mut processor, &crashes).await?; } - if let Some(queue) = &self.config.input_queue { - let callback = CallbackImpl::new(queue.clone(), processor); - self.poller.run(callback).await?; + if self.config.check_queue { + if let Some(queue) = &self.config.input_queue { + let callback = CallbackImpl::new(queue.clone(), processor); + self.poller.run(callback).await?; + } } Ok(()) } @@ -104,6 +136,20 @@ impl<'a> GenericReportProcessor<'a> { } } + async fn test_local(&self, input: PathBuf) -> Result<()> { + info!("generating crash report for: {}", input.display()); + let result = self.test_input(None, &input).await?; + result + .save_local( + &self.config.unique_reports, + &self.config.reports, + &self.config.no_repro, + ) + .await?; + + Ok(()) + } + pub async fn test_input( &self, input_url: Option, From 3d1961139f8f8633f05d732a5afd7c8cae4dba66 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 22 Dec 2020 16:10:20 -0500 Subject: [PATCH 07/30] remove dup disable check queue --- src/agent/onefuzz-agent/src/local/generic_crash_report.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 8627a2afcd..ffe3a9eae7 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -111,11 +111,6 @@ pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { .takes_value(false) .long("disable_check_debugger"), ) - .arg( - Arg::with_name("disable_check_queue") - .takes_value(false) - .long("disable_check_queue"), - ) } pub fn args(name: &'static str) -> App<'static, 'static> { From 22f82624151ed8d98691c2cccd97f96298819b78 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 22 Dec 2020 16:18:48 -0500 Subject: [PATCH 08/30] only save reports if they don't already exist --- .../src/tasks/report/crash_report.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index a27a7c4f4c..d2e8dd10df 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -5,6 +5,7 @@ use anyhow::Result; use onefuzz::{ asan::AsanLog, blob::{BlobClient, BlobContainerUrl, BlobUrl}, + fs::exists, syncdir::SyncedDir, telemetry::Event::{new_report, new_unable_to_reproduce, new_unique_report}, }; @@ -105,20 +106,26 @@ impl CrashTestResult { ) -> Result<()> { match self { Self::CrashReport(report) => { - let data = serde_json::to_vec(&report)?; let unique_path = unique_reports.path.join(report.blob_name()); - fs::write(unique_path, &data).await?; + let data = serde_json::to_vec(&report)?; + if !exists(&unique_path).await? { + fs::write(unique_path, &data).await?; + } if let Some(reports) = reports { let report_path = reports.path.join(report.blob_name()); - fs::write(report_path, &data).await?; + if !exists(&report_path).await? { + fs::write(report_path, &data).await?; + } } } Self::NoRepro(report) => { if let Some(no_repro) = no_repro { - let data = serde_json::to_vec(&report)?; let no_repro_path = no_repro.path.join(report.blob_name()); - fs::write(no_repro_path, &data).await?; + if !exists(&no_repro_path).await? { + let data = serde_json::to_vec(&report)?; + fs::write(no_repro_path, &data).await?; + } } } } From 8815d31a33d6a9a2bc39e1f8451f0b539fc31efc Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 23 Dec 2020 21:48:49 -0500 Subject: [PATCH 09/30] continued dev --- .../src/debug/libfuzzer_merge.rs | 34 +-- src/agent/onefuzz-agent/src/local/cmd.rs | 9 +- src/agent/onefuzz-agent/src/local/common.rs | 83 ++++-- .../src/local/generic_crash_report.rs | 24 +- .../src/local/generic_generator.rs | 115 ++++++++ .../onefuzz-agent/src/local/libfuzzer.rs | 8 +- .../src/local/libfuzzer_coverage.rs | 15 +- .../src/local/libfuzzer_crash_report.rs | 16 +- .../onefuzz-agent/src/local/libfuzzer_fuzz.rs | 17 +- src/agent/onefuzz-agent/src/local/mod.rs | 1 + src/agent/onefuzz-agent/src/tasks/config.rs | 8 +- .../onefuzz-agent/src/tasks/fuzz/generator.rs | 248 +++++++++--------- 12 files changed, 377 insertions(+), 201 deletions(-) create mode 100644 src/agent/onefuzz-agent/src/local/generic_generator.rs diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs index faaae3f800..68b8d9bd57 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs @@ -2,7 +2,9 @@ // Licensed under the MIT License. use crate::{ - local::common::build_common_config, + local::common::{ + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + }, tasks::{ merge::libfuzzer_merge::{merge_inputs, Config}, utils::parse_key_value, @@ -14,16 +16,12 @@ use onefuzz::syncdir::SyncedDir; use std::{collections::HashMap, path::PathBuf, sync::Arc}; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + let inputs = value_t!(args, "inputs", String)?; let unique_inputs = value_t!(args, "unique_inputs", String)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } let common = build_common_config(args)?; let config = Arc::new(Config { @@ -49,23 +47,13 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { } pub fn args(name: &'static str) -> App<'static, 'static> { - SubCommand::with_name(name) - .about("execute a local-only libfuzzer merge task") - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name("inputs").takes_value(true).required(true)) + let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer merge task"); + + app = add_cmd_options(CmdType::Target, true, true, true, app); + app.arg(Arg::with_name("inputs").takes_value(true).required(true)) .arg( Arg::with_name("unique_inputs") .takes_value(true) .required(true), ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) } diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs index 85ba1e519c..38fcb71221 100644 --- a/src/agent/onefuzz-agent/src/local/cmd.rs +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -5,8 +5,8 @@ use anyhow::Result; use clap::{App, SubCommand}; use crate::local::{ - common::add_common_config, generic_crash_report, libfuzzer, libfuzzer_coverage, - libfuzzer_crash_report, libfuzzer_fuzz, + common::add_common_config, generic_crash_report, generic_generator, libfuzzer, + libfuzzer_coverage, libfuzzer_crash_report, libfuzzer_fuzz, }; const LIBFUZZER: &str = "libfuzzer"; @@ -14,6 +14,7 @@ const LIBFUZZER_FUZZ: &str = "libfuzzer-fuzz"; const LIBFUZZER_CRASH_REPORT: &str = "libfuzzer-crash-report"; const LIBFUZZER_COVERAGE: &str = "libfuzzer-coverage"; const GENERIC_CRASH_REPORT: &str = "generic-crash-report"; +const GENERIC_GENERATOR: &str = "generic-generator"; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { @@ -22,6 +23,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub).await, + (GENERIC_GENERATOR, Some(sub)) => generic_generator::run(sub).await, _ => { anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); } @@ -42,4 +44,7 @@ pub fn args(name: &str) -> App<'static, 'static> { .subcommand(add_common_config(generic_crash_report::args( GENERIC_CRASH_REPORT, ))) + .subcommand(add_common_config(generic_generator::args( + GENERIC_GENERATOR, + ))) } diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 35b121b63f..2ed32673a5 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -3,11 +3,9 @@ use crate::tasks::utils::parse_key_value; use anyhow::Result; use clap::{App, Arg, ArgMatches}; use std::collections::HashMap; +use std::path::PathBuf; use uuid::Uuid; -pub const TARGET_EXE: &str = "target_exe"; -const TARGET_ENV: &str = "target_env"; -pub const TARGET_OPTIONS: &str = "target_options"; pub const INPUTS_DIR: &str = "inputs_dir"; pub const CRASHES_DIR: &str = "crashes_dir"; pub const TARGET_WORKERS: &str = "target_workers"; @@ -18,44 +16,99 @@ pub const CHECK_RETRY_COUNT: &str = "check_retry_count"; pub const DISABLE_CHECK_QUEUE: &str = "disable_check_queue"; pub const UNIQUE_REPORTS_DIR: &str = "unique_reports_dir"; pub const COVERAGE_DIR: &str = "coverage_dir"; +pub const READONLY_INPUTS: &str = "readonly_inputs_dir"; +pub const CHECK_ASAN_LOG: &str = "check_asan_log"; + +pub const TARGET_EXE: &str = "target_exe"; +pub const TARGET_ENV: &str = "target_env"; +pub const TARGET_OPTIONS: &str = "target_options"; +pub const SUPERVISOR_EXE: &str = "supervisor_exe"; +pub const SUPERVISOR_ENV: &str = "supervisor_env"; +pub const SUPERVISOR_OPTIONS: &str = "supervisor_options"; +pub const GENERATOR_EXE: &str = "generator_exe"; +pub const GENERATOR_ENV: &str = "generator_env"; +pub const GENERATOR_OPTIONS: &str = "generator_options"; -pub fn add_target_cmd_options( +pub enum CmdType { + Target, + Generator, + Supervisor, +} + +pub fn add_cmd_options( + cmd_type: CmdType, exe: bool, arg: bool, env: bool, mut app: App<'static, 'static>, ) -> App<'static, 'static> { + let (exe_name, env_name, arg_name) = match cmd_type { + Target => (TARGET_EXE, TARGET_ENV, TARGET_OPTIONS), + Supervisor => (SUPERVISOR_EXE, SUPERVISOR_ENV, SUPERVISOR_OPTIONS), + Generator => (GENERATOR_EXE, GENERATOR_ENV, GENERATOR_OPTIONS), + }; + if exe { - app = app.arg(Arg::with_name(TARGET_EXE).takes_value(true).required(true)); + app = app.arg(Arg::with_name(exe_name).takes_value(true).required(true)); } if arg { app = app.arg( - Arg::with_name(TARGET_ENV) - .long(TARGET_ENV) + Arg::with_name(env_name) + .long(env_name) .takes_value(true) .multiple(true), ) } if env { app = app.arg( - Arg::with_name(TARGET_OPTIONS) - .long(TARGET_OPTIONS) + Arg::with_name(arg_name) + .long(arg_name) .takes_value(true) .multiple(true) .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set target_env first"), + .help("Supports hyphens. Recommendation: Set env first"), ) } app } -pub fn get_target_env(args: &clap::ArgMatches<'_>) -> Result> { - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { +pub fn get_cmd_exe(cmd_type: CmdType, args: &clap::ArgMatches<'_>) -> Result { + let name = match cmd_type { + Target => TARGET_EXE, + Supervisor => SUPERVISOR_EXE, + Generator => GENERATOR_EXE, + }; + + let exe = value_t!(args, name, String)?; + Ok(exe) +} + +pub fn get_cmd_arg(cmd_type: CmdType, args: &clap::ArgMatches<'_>) -> Vec { + let name = match cmd_type { + Target => TARGET_OPTIONS, + Supervisor => SUPERVISOR_OPTIONS, + Generator => GENERATOR_OPTIONS, + }; + + args.values_of_lossy(name).unwrap_or_default() +} + +pub fn get_cmd_env( + cmd_type: CmdType, + args: &clap::ArgMatches<'_>, +) -> Result> { + let env_name = match cmd_type { + Target => TARGET_ENV, + Supervisor => SUPERVISOR_ENV, + Generator => GENERATOR_ENV, + }; + + let mut env = HashMap::new(); + for opt in args.values_of_lossy(env_name).unwrap_or_default() { let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); + env.insert(k, v); } - Ok(target_env) + Ok(env) } pub fn add_common_config(app: App<'static, 'static>) -> App<'static, 'static> { diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index ffe3a9eae7..9b2c9a7a54 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -3,9 +3,9 @@ use crate::{ local::common::{ - add_target_cmd_options, build_common_config, get_target_env, CHECK_RETRY_COUNT, - CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_EXE, TARGET_OPTIONS, - TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, + REPORTS_DIR, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::report::generic::{Config, ReportTask}, }; @@ -14,7 +14,10 @@ use clap::{App, Arg, SubCommand}; use std::path::PathBuf; pub async fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { - let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + let crashes = Some(value_t!(args, CRASHES_DIR, PathBuf)?.into()); let reports = if args.is_present(REPORTS_DIR) { Some(value_t!(args, REPORTS_DIR, PathBuf)?).map(|x| x.into()) @@ -28,14 +31,11 @@ pub async fn build_report_config(args: &clap::ArgMatches<'_>) -> Result }; let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); - let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); - let target_env = get_target_env(args)?; - let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; let check_queue = !args.is_present(DISABLE_CHECK_QUEUE); - let check_asan_log = args.is_present("check_asan_log"); + let check_asan_log = args.is_present(CHECK_ASAN_LOG); let check_debugger = !args.is_present("disable_check_debugger"); let common = build_common_config(args)?; @@ -88,7 +88,7 @@ pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { Arg::with_name(TARGET_TIMEOUT) .takes_value(true) .long(TARGET_TIMEOUT) - .default_value("5"), + .default_value("30"), ) .arg( Arg::with_name(CHECK_RETRY_COUNT) @@ -102,9 +102,9 @@ pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { .long(DISABLE_CHECK_QUEUE), ) .arg( - Arg::with_name("check_asan_log") + Arg::with_name(CHECK_ASAN_LOG) .takes_value(false) - .long("check_asan_log"), + .long(CHECK_ASAN_LOG), ) .arg( Arg::with_name("disable_check_debugger") @@ -115,6 +115,6 @@ pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { pub fn args(name: &'static str) -> App<'static, 'static> { let mut app = SubCommand::with_name(name).about("execute a local-only generic crash report"); - app = add_target_cmd_options(true, true, true, app); + app = add_cmd_options(CmdType::Target, true, true, true, app); add_report_options(app) } diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs new file mode 100644 index 0000000000..55a1b0602e --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, INPUTS_DIR, READONLY_INPUTS, + TARGET_TIMEOUT, + }, + tasks::fuzz::generator::{Config, GeneratorTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +const TOOLS_DIR: &str = "tools_dir"; +const RENAME_OUTPUT: &str = "rename_output"; + +pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { + let crashes = value_t!(args, CRASHES_DIR, PathBuf)?.into(); + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_options = get_cmd_arg(CmdType::Target, args); + let target_env = get_cmd_env(CmdType::Target, args)?; + + let generator_exe = get_cmd_exe(CmdType::Generator, args)?; + let generator_options = get_cmd_arg(CmdType::Generator, args); + let generator_env = get_cmd_env(CmdType::Generator, args)?; + + let readonly_inputs = values_t!(args, READONLY_INPUTS, PathBuf)? + .iter() + .map(|x| x.to_owned().into()) + .collect(); + + let rename_output = args.is_present(RENAME_OUTPUT); + let check_asan_log = args.is_present(CHECK_ASAN_LOG); + let check_debugger = args.is_present("disable_check_debugger"); + let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; + let target_timeout = Some(value_t!(args, TARGET_TIMEOUT, u64)?); + + let tools = if args.is_present(TOOLS_DIR) { + Some(value_t!(args, TOOLS_DIR, PathBuf)?.into()) + } else { + None + }; + + let ensemble_sync_delay = None; + let common = build_common_config(args)?; + let config = Config { + tools, + generator_exe, + generator_env, + generator_options, + target_exe, + target_env, + target_options, + target_timeout, + readonly_inputs, + crashes, + ensemble_sync_delay, + check_asan_log, + check_debugger, + check_retry_count, + rename_output, + common, + }; + + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_fuzz_config(args)?; + GeneratorTask::new(config).local_run().await +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("execute a local-only generator fuzzing task"); + + app = add_cmd_options(CmdType::Generator, true, true, true, app); + app = add_cmd_options(CmdType::Target, true, true, true, app); + app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) + .arg( + Arg::with_name(READONLY_INPUTS) + .takes_value(true) + .required(true) + .multiple(true), + ) + .arg(Arg::with_name(TOOLS_DIR).takes_value(true)) + .arg( + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + ) + .arg( + Arg::with_name(CHECK_ASAN_LOG) + .takes_value(false) + .long(CHECK_ASAN_LOG), + ) + .arg( + Arg::with_name(RENAME_OUTPUT) + .takes_value(false) + .long(RENAME_OUTPUT), + ) + .arg( + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT) + .default_value("30"), + ) + .arg( + Arg::with_name("disable_check_debugger") + .takes_value(false) + .long("disable_check_debugger"), + ) +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index 97e08b4bc5..f17896782e 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -4,9 +4,9 @@ use crate::{ local::{ common::{ - add_target_cmd_options, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, - DISABLE_CHECK_QUEUE, INPUTS_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, - TARGET_WORKERS, UNIQUE_REPORTS_DIR, + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, DISABLE_CHECK_QUEUE, INPUTS_DIR, + NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, UNIQUE_REPORTS_DIR, }, libfuzzer_coverage::build_coverage_config, libfuzzer_crash_report::build_report_config, @@ -45,7 +45,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub fn args(name: &'static str) -> App<'static, 'static> { let mut app = SubCommand::with_name(name).about("run a local libfuzzer & crash reporting task"); - app = add_target_cmd_options(true, true, true, app); + app = add_cmd_options(CmdType::Target, true, true, true, app); app.arg(Arg::with_name(INPUTS_DIR).takes_value(true).required(true)) .arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 5ff6d4a8eb..40c861c549 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -3,8 +3,8 @@ use crate::{ local::common::{ - add_target_cmd_options, build_common_config, get_target_env, COVERAGE_DIR, INPUTS_DIR, - TARGET_EXE, TARGET_OPTIONS, + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, TARGET_EXE, TARGET_OPTIONS, }, tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, }; @@ -12,10 +12,10 @@ use anyhow::Result; use clap::{App, Arg, SubCommand}; use std::path::PathBuf; -const READONLY_INPUTS: &str = "readonly_inputs_dir"; - pub fn build_coverage_config(args: &clap::ArgMatches<'_>, use_inputs: bool) -> Result { - let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); let readonly_inputs = if use_inputs { vec![value_t!(args, INPUTS_DIR, PathBuf)?.into()] @@ -27,9 +27,6 @@ pub fn build_coverage_config(args: &clap::ArgMatches<'_>, use_inputs: bool) -> R }; let coverage = value_t!(args, COVERAGE_DIR, PathBuf)?.into(); - let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); - - let target_env = get_target_env(args)?; let common = build_common_config(args)?; let config = Config { @@ -55,7 +52,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub fn args(name: &'static str) -> App<'static, 'static> { let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer coverage task"); - app = add_target_cmd_options(true, true, true, app); + app = add_cmd_options(CmdType::Target, true, true, true, app); app.arg( Arg::with_name(COVERAGE_DIR) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index a603f0f506..514eac720b 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -3,9 +3,9 @@ use crate::{ local::common::{ - add_target_cmd_options, build_common_config, get_target_env, CHECK_RETRY_COUNT, - CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_EXE, TARGET_OPTIONS, - TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_EXE, + TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::report::libfuzzer_report::{Config, ReportTask}, }; @@ -14,7 +14,10 @@ use clap::{App, Arg, SubCommand}; use std::path::PathBuf; pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { - let target_exe = value_t!(args, TARGET_EXE, PathBuf)?; + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + let crashes = Some(value_t!(args, CRASHES_DIR, PathBuf)?.into()); let reports = if args.is_present(REPORTS_DIR) { Some(value_t!(args, REPORTS_DIR, PathBuf)?).map(|x| x.into()) @@ -28,9 +31,6 @@ pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { }; let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); - let target_options = args.values_of_lossy(TARGET_OPTIONS).unwrap_or_default(); - let target_env = get_target_env(args)?; - let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; @@ -100,6 +100,6 @@ pub fn args(name: &'static str) -> App<'static, 'static> { let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer crash report task"); - app = add_target_cmd_options(true, true, true, app); + app = add_cmd_options(CmdType::Target, true, true, true, app); add_report_options(app) } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index dd3eb1af0e..8e11efd2a4 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -3,8 +3,8 @@ use crate::{ local::common::{ - add_target_cmd_options, build_common_config, get_target_env, CRASHES_DIR, INPUTS_DIR, - TARGET_WORKERS, + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + CRASHES_DIR, INPUTS_DIR, TARGET_WORKERS, }, tasks::fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, }; @@ -15,10 +15,12 @@ use std::path::PathBuf; pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { let crashes = value_t!(args, CRASHES_DIR, PathBuf)?.into(); let inputs = value_t!(args, INPUTS_DIR, PathBuf)?.into(); - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); + + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + let target_workers = value_t!(args, "target_workers", u64).unwrap_or_default(); - let target_env = get_target_env(args)?; let readonly_inputs = None; let ensemble_sync_delay = None; @@ -44,10 +46,9 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { } pub fn args(name: &'static str) -> App<'static, 'static> { - let mut app = - SubCommand::with_name(name).about("execute a local-only libfuzzer crash report task"); + let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer fuzzing task"); - app = add_target_cmd_options(true, true, true, app); + app = add_cmd_options(CmdType::Target, true, true, true, app); app.arg(Arg::with_name(INPUTS_DIR).takes_value(true).required(true)) .arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) .arg( diff --git a/src/agent/onefuzz-agent/src/local/mod.rs b/src/agent/onefuzz-agent/src/local/mod.rs index 953ff124eb..1840ddd492 100644 --- a/src/agent/onefuzz-agent/src/local/mod.rs +++ b/src/agent/onefuzz-agent/src/local/mod.rs @@ -4,6 +4,7 @@ pub mod cmd; pub mod common; pub mod generic_crash_report; +pub mod generic_generator; pub mod libfuzzer; pub mod libfuzzer_coverage; pub mod libfuzzer_crash_report; diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 3e0fa25077..15ba395b07 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -66,7 +66,7 @@ pub enum Config { GenericAnalysis(analysis::generic::Config), #[serde(alias = "generic_generator")] - GenericGenerator(fuzz::generator::GeneratorConfig), + GenericGenerator(fuzz::generator::Config), #[serde(alias = "generic_supervisor")] GenericSupervisor(fuzz::supervisor::SupervisorConfig), @@ -156,7 +156,11 @@ impl Config { } Config::LibFuzzerMerge(config) => merge::libfuzzer_merge::spawn(Arc::new(config)).await, Config::GenericAnalysis(config) => analysis::generic::spawn(config).await, - Config::GenericGenerator(config) => fuzz::generator::spawn(Arc::new(config)).await, + Config::GenericGenerator(config) => { + fuzz::generator::GeneratorTask::new(config) + .managed_run() + .await + } Config::GenericSupervisor(config) => fuzz::supervisor::spawn(config).await, Config::GenericMerge(config) => merge::generic::spawn(Arc::new(config)).await, Config::GenericReport(config) => { diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index 6426527e25..f6ca0082e9 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -2,7 +2,7 @@ // Licensed under the MIT License. use crate::tasks::{config::CommonConfig, heartbeat::*, utils}; -use anyhow::{Error, Result}; +use anyhow::Result; use futures::stream::StreamExt; use onefuzz::{ expand::Expand, @@ -18,8 +18,8 @@ use std::{ ffi::OsString, path::{Path, PathBuf}, process::Stdio, - sync::Arc, }; +use tempfile::tempdir; use tokio::{fs, process::Command}; fn default_bool_true() -> bool { @@ -27,13 +27,13 @@ fn default_bool_true() -> bool { } #[derive(Debug, Deserialize, Clone)] -pub struct GeneratorConfig { +pub struct Config { pub generator_exe: String, pub generator_env: HashMap, pub generator_options: Vec, pub readonly_inputs: Vec, pub crashes: SyncedDir, - pub tools: SyncedDir, + pub tools: Option, pub target_exe: PathBuf, pub target_env: HashMap, @@ -51,133 +51,145 @@ pub struct GeneratorConfig { pub common: CommonConfig, } -pub async fn spawn(config: Arc) -> Result<(), Error> { - config.crashes.init().await?; - config.tools.init_pull().await?; +pub struct GeneratorTask { + config: Config, +} + +impl GeneratorTask { + pub fn new(config: Config) -> Self { + Self { config } + } - set_executable(&config.tools.path).await?; - let hb_client = config.common.init_heartbeat().await?; + pub async fn local_run(&self) -> Result<()> { + self.config.crashes.init().await?; + for dir in &self.config.readonly_inputs { + dir.init().await?; + } - for dir in &config.readonly_inputs { - dir.init_pull().await?; + self.fuzzing_loop(None).await } - let sync_task = continuous_sync(&config.readonly_inputs, Pull, config.ensemble_sync_delay); - let crash_dir_monitor = config.crashes.monitor_results(new_result); - let tester = Tester::new( - &config.target_exe, - &config.target_options, - &config.target_env, - &config.target_timeout, - config.check_asan_log, - false, - config.check_debugger, - config.check_retry_count, - ); - let inputs: Vec<_> = config.readonly_inputs.iter().map(|x| &x.path).collect(); - let fuzzing_monitor = start_fuzzing(&config, inputs, tester, hb_client); - futures::try_join!(fuzzing_monitor, sync_task, crash_dir_monitor)?; - Ok(()) -} + pub async fn managed_run(&self) -> Result<()> { + self.config.crashes.init().await?; + if let Some(tools) = &self.config.tools { + tools.init_pull().await?; + set_executable(&tools.path).await?; + } + + let hb_client = self.config.common.init_heartbeat().await?; + + for dir in &self.config.readonly_inputs { + dir.init_pull().await?; + } + + let sync_task = continuous_sync( + &self.config.readonly_inputs, + Pull, + self.config.ensemble_sync_delay, + ); + + let crash_dir_monitor = self.config.crashes.monitor_results(new_result); + + let fuzzer = self.fuzzing_loop(hb_client); + + futures::try_join!(fuzzer, sync_task, crash_dir_monitor)?; + Ok(()) + } -async fn generate_input( - generator_exe: &str, - generator_env: &HashMap, - generator_options: &[String], - tools_dir: impl AsRef, - corpus_dir: impl AsRef, - output_dir: impl AsRef, -) -> Result<()> { - let mut expand = Expand::new(); - expand - .generated_inputs(&output_dir) - .input_corpus(&corpus_dir) - .generator_exe(&generator_exe) - .generator_options(&generator_options) - .tools_dir(&tools_dir); - - utils::reset_tmp_dir(&output_dir).await?; - - let generator_path = Expand::new() - .tools_dir(tools_dir.as_ref()) - .evaluate_value(generator_exe)?; - - let mut generator = Command::new(&generator_path); - generator - .kill_on_drop(true) - .env_remove("RUST_LOG") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()); - - for arg in expand.evaluate(generator_options)? { - generator.arg(arg); + async fn fuzzing_loop(&self, heartbeat_client: Option) -> Result<()> { + let tester = Tester::new( + &self.config.target_exe, + &self.config.target_options, + &self.config.target_env, + &self.config.target_timeout, + self.config.check_asan_log, + false, + self.config.check_debugger, + self.config.check_retry_count, + ); + + loop { + for corpus_dir in &self.config.readonly_inputs { + heartbeat_client.alive(); + let corpus_dir = &corpus_dir.path; + let generated_inputs = tempdir()?; + let generated_inputs_path = generated_inputs.path(); + + self.generate_inputs(corpus_dir, &generated_inputs_path) + .await?; + self.test_inputs(&generated_inputs_path, &tester).await?; + } + } } - for (k, v) in generator_env { - generator.env(k, expand.evaluate_value(v)?); + async fn test_inputs( + &self, + generated_inputs: impl AsRef, + tester: &Tester<'_>, + ) -> Result<()> { + let mut read_dir = fs::read_dir(generated_inputs).await?; + while let Some(file) = read_dir.next().await { + let file = file?; + + verbose!("testing input: {:?}", file); + + let destination_file = if self.config.rename_output { + let hash = sha256::digest_file(file.path()).await?; + OsString::from(hash) + } else { + file.file_name() + }; + + let destination_file = self.config.crashes.path.join(destination_file); + if tester.is_crash(file.path()).await? { + info!("Crash found, path = {}", file.path().display()); + fs::rename(file.path(), &destination_file).await?; + } + } + Ok(()) } - info!("Generating test cases with {:?}", generator); - let output = generator.spawn()?.wait_with_output().await?; + async fn generate_inputs( + &self, + corpus_dir: impl AsRef, + output_dir: impl AsRef, + ) -> Result<()> { + let mut expand = Expand::new(); + expand + .generated_inputs(&output_dir) + .input_corpus(&corpus_dir) + .generator_exe(&self.config.generator_exe) + .generator_options(&self.config.generator_options); + + if let Some(tools) = &self.config.tools { + expand.tools_dir(&tools.path); + } - info!("Test case generation result {:?}", output); - Ok(()) -} + utils::reset_tmp_dir(&output_dir).await?; -async fn start_fuzzing<'a>( - config: &GeneratorConfig, - corpus_dirs: Vec>, - tester: Tester<'a>, - heartbeat_client: Option, -) -> Result<()> { - let generator_tmp = "generator_tmp"; - - info!("Starting generator fuzzing loop"); - - loop { - heartbeat_client.alive(); - - for corpus_dir in &corpus_dirs { - let corpus_dir = corpus_dir.as_ref(); - - generate_input( - &config.generator_exe, - &config.generator_env, - &config.generator_options, - &config.tools.path, - corpus_dir, - generator_tmp, - ) - .await?; - - let mut read_dir = fs::read_dir(generator_tmp).await?; - while let Some(file) = read_dir.next().await { - verbose!("Processing file {:?}", file); - let file = file?; - - let destination_file = if config.rename_output { - let hash = sha256::digest_file(file.path()).await?; - OsString::from(hash) - } else { - file.file_name() - }; - - let destination_file = config.crashes.path.join(destination_file); - if tester.is_crash(file.path()).await? { - info!("Crash found, path = {}", file.path().display()); - - if let Err(err) = fs::rename(file.path(), &destination_file).await { - warn!("Unable to move file {:?} : {:?}", file.path(), err); - } - } - } + let generator_path = expand.evaluate_value(&self.config.generator_exe)?; + + let mut generator = Command::new(&generator_path); + generator + .kill_on_drop(true) + .env_remove("RUST_LOG") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + + for arg in expand.evaluate(&self.config.generator_options)? { + generator.arg(arg); + } - verbose!( - "Tested generated inputs for corpus = {}", - corpus_dir.display() - ); + for (k, v) in &self.config.generator_env { + generator.env(k, expand.evaluate_value(v)?); } + + info!("Generating test cases with {:?}", generator); + let output = generator.spawn()?.wait_with_output().await?; + + info!("Test case generation result {:?}", output); + Ok(()) } } From fe1466bfa725b686fd1dc98d861359ef183af5d8 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 23 Dec 2020 21:52:09 -0500 Subject: [PATCH 10/30] continued dev --- .../src/debug/libfuzzer_merge.rs | 7 ++--- src/agent/onefuzz-agent/src/local/common.rs | 26 +++++++++---------- .../src/local/generic_generator.rs | 3 +-- .../onefuzz-agent/src/local/libfuzzer.rs | 6 ++--- .../src/local/libfuzzer_coverage.rs | 2 +- .../src/local/libfuzzer_crash_report.rs | 4 +-- 6 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs index 68b8d9bd57..d4295acca5 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs @@ -5,15 +5,12 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, }, - tasks::{ - merge::libfuzzer_merge::{merge_inputs, Config}, - utils::parse_key_value, - }, + tasks::merge::libfuzzer_merge::{merge_inputs, Config}, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; use onefuzz::syncdir::SyncedDir; -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::sync::Arc; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 2ed32673a5..4b0bdd2f49 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -3,7 +3,7 @@ use crate::tasks::utils::parse_key_value; use anyhow::Result; use clap::{App, Arg, ArgMatches}; use std::collections::HashMap; -use std::path::PathBuf; + use uuid::Uuid; pub const INPUTS_DIR: &str = "inputs_dir"; @@ -43,9 +43,9 @@ pub fn add_cmd_options( mut app: App<'static, 'static>, ) -> App<'static, 'static> { let (exe_name, env_name, arg_name) = match cmd_type { - Target => (TARGET_EXE, TARGET_ENV, TARGET_OPTIONS), - Supervisor => (SUPERVISOR_EXE, SUPERVISOR_ENV, SUPERVISOR_OPTIONS), - Generator => (GENERATOR_EXE, GENERATOR_ENV, GENERATOR_OPTIONS), + CmdType::Target => (TARGET_EXE, TARGET_ENV, TARGET_OPTIONS), + CmdType::Supervisor => (SUPERVISOR_EXE, SUPERVISOR_ENV, SUPERVISOR_OPTIONS), + CmdType::Generator => (GENERATOR_EXE, GENERATOR_ENV, GENERATOR_OPTIONS), }; if exe { @@ -74,9 +74,9 @@ pub fn add_cmd_options( pub fn get_cmd_exe(cmd_type: CmdType, args: &clap::ArgMatches<'_>) -> Result { let name = match cmd_type { - Target => TARGET_EXE, - Supervisor => SUPERVISOR_EXE, - Generator => GENERATOR_EXE, + CmdType::Target => TARGET_EXE, + CmdType::Supervisor => SUPERVISOR_EXE, + CmdType::Generator => GENERATOR_EXE, }; let exe = value_t!(args, name, String)?; @@ -85,9 +85,9 @@ pub fn get_cmd_exe(cmd_type: CmdType, args: &clap::ArgMatches<'_>) -> Result) -> Vec { let name = match cmd_type { - Target => TARGET_OPTIONS, - Supervisor => SUPERVISOR_OPTIONS, - Generator => GENERATOR_OPTIONS, + CmdType::Target => TARGET_OPTIONS, + CmdType::Supervisor => SUPERVISOR_OPTIONS, + CmdType::Generator => GENERATOR_OPTIONS, }; args.values_of_lossy(name).unwrap_or_default() @@ -98,9 +98,9 @@ pub fn get_cmd_env( args: &clap::ArgMatches<'_>, ) -> Result> { let env_name = match cmd_type { - Target => TARGET_ENV, - Supervisor => SUPERVISOR_ENV, - Generator => GENERATOR_ENV, + CmdType::Target => TARGET_ENV, + CmdType::Supervisor => SUPERVISOR_ENV, + CmdType::Generator => GENERATOR_ENV, }; let mut env = HashMap::new(); diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index 55a1b0602e..888d97b90c 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -4,8 +4,7 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, INPUTS_DIR, READONLY_INPUTS, - TARGET_TIMEOUT, + CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, READONLY_INPUTS, TARGET_TIMEOUT, }, tasks::fuzz::generator::{Config, GeneratorTask}, }; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index f17896782e..904ec0b149 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -4,9 +4,9 @@ use crate::{ local::{ common::{ - add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, DISABLE_CHECK_QUEUE, INPUTS_DIR, - NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, UNIQUE_REPORTS_DIR, + add_cmd_options, CmdType, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, + DISABLE_CHECK_QUEUE, INPUTS_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, + TARGET_WORKERS, UNIQUE_REPORTS_DIR, }, libfuzzer_coverage::build_coverage_config, libfuzzer_crash_report::build_report_config, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 40c861c549..e4f14fa789 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, TARGET_EXE, TARGET_OPTIONS, + COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, }, tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, }; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index 514eac720b..f928a8ff17 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -4,8 +4,8 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_EXE, - TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, + TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::report::libfuzzer_report::{Config, ReportTask}, }; From b8c6bd19f9d88b5a7e1eb173afd5bd8b32204497 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 24 Dec 2020 00:03:29 -0500 Subject: [PATCH 11/30] continued dev --- src/agent/onefuzz-agent/src/local/cmd.rs | 5 +- src/agent/onefuzz-agent/src/local/common.rs | 8 +- .../src/local/generic_crash_report.rs | 4 +- .../src/local/generic_generator.rs | 6 +- .../onefuzz-agent/src/local/libfuzzer.rs | 76 +++++---------- .../src/local/libfuzzer_coverage.rs | 58 +++++++++--- .../src/local/libfuzzer_crash_report.rs | 90 +++++++++--------- .../onefuzz-agent/src/local/libfuzzer_fuzz.rs | 45 ++++++--- src/agent/onefuzz-agent/src/local/mod.rs | 1 + src/agent/onefuzz-agent/src/local/radamsa.rs | 92 +++++++++++++++++++ .../onefuzz-agent/src/tasks/report/generic.rs | 2 +- 11 files changed, 254 insertions(+), 133 deletions(-) create mode 100644 src/agent/onefuzz-agent/src/local/radamsa.rs diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs index 38fcb71221..4bf17e4005 100644 --- a/src/agent/onefuzz-agent/src/local/cmd.rs +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -6,9 +6,10 @@ use clap::{App, SubCommand}; use crate::local::{ common::add_common_config, generic_crash_report, generic_generator, libfuzzer, - libfuzzer_coverage, libfuzzer_crash_report, libfuzzer_fuzz, + libfuzzer_coverage, libfuzzer_crash_report, libfuzzer_fuzz, radamsa, }; +const RADAMSA: &str = "radamsa"; const LIBFUZZER: &str = "libfuzzer"; const LIBFUZZER_FUZZ: &str = "libfuzzer-fuzz"; const LIBFUZZER_CRASH_REPORT: &str = "libfuzzer-crash-report"; @@ -18,6 +19,7 @@ const GENERIC_GENERATOR: &str = "generic-generator"; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { + (RADAMSA, Some(sub)) => radamsa::run(sub).await, (LIBFUZZER, Some(sub)) => libfuzzer::run(sub).await, (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, @@ -33,6 +35,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub fn args(name: &str) -> App<'static, 'static> { SubCommand::with_name(name) .about("pre-release local fuzzing") + .subcommand(add_common_config(radamsa::args(RADAMSA))) .subcommand(add_common_config(libfuzzer::args(LIBFUZZER))) .subcommand(add_common_config(libfuzzer_fuzz::args(LIBFUZZER_FUZZ))) .subcommand(add_common_config(libfuzzer_coverage::args( diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 4b0bdd2f49..d2c89d33ab 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -2,7 +2,7 @@ use crate::tasks::config::CommonConfig; use crate::tasks::utils::parse_key_value; use anyhow::Result; use clap::{App, Arg, ArgMatches}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use uuid::Uuid; @@ -18,6 +18,8 @@ pub const UNIQUE_REPORTS_DIR: &str = "unique_reports_dir"; pub const COVERAGE_DIR: &str = "coverage_dir"; pub const READONLY_INPUTS: &str = "readonly_inputs_dir"; pub const CHECK_ASAN_LOG: &str = "check_asan_log"; +pub const TOOLS_DIR: &str = "tools_dir"; +pub const RENAME_OUTPUT: &str = "rename_output"; pub const TARGET_EXE: &str = "target_exe"; pub const TARGET_ENV: &str = "target_env"; @@ -51,7 +53,7 @@ pub fn add_cmd_options( if exe { app = app.arg(Arg::with_name(exe_name).takes_value(true).required(true)); } - if arg { + if env { app = app.arg( Arg::with_name(env_name) .long(env_name) @@ -59,7 +61,7 @@ pub fn add_cmd_options( .multiple(true), ) } - if env { + if arg { app = app.arg( Arg::with_name(arg_name) .long(arg_name) diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 9b2c9a7a54..88ab63fe4f 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -13,7 +13,7 @@ use anyhow::Result; use clap::{App, Arg, SubCommand}; use std::path::PathBuf; -pub async fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { +pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; let target_options = get_cmd_arg(CmdType::Target, args); @@ -61,7 +61,7 @@ pub async fn build_report_config(args: &clap::ArgMatches<'_>) -> Result } pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { - let config = build_report_config(args).await?; + let config = build_report_config(args)?; ReportTask::new(&config).local_run().await } diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index 888d97b90c..d60770c96f 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -4,7 +4,8 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, READONLY_INPUTS, TARGET_TIMEOUT, + CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, READONLY_INPUTS, RENAME_OUTPUT, + TARGET_TIMEOUT, TOOLS_DIR, }, tasks::fuzz::generator::{Config, GeneratorTask}, }; @@ -12,9 +13,6 @@ use anyhow::Result; use clap::{App, Arg, SubCommand}; use std::path::PathBuf; -const TOOLS_DIR: &str = "tools_dir"; -const RENAME_OUTPUT: &str = "rename_output"; - pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { let crashes = value_t!(args, CRASHES_DIR, PathBuf)?.into(); let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index 904ec0b149..6fff64824f 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -4,13 +4,12 @@ use crate::{ local::{ common::{ - add_cmd_options, CmdType, CHECK_RETRY_COUNT, COVERAGE_DIR, CRASHES_DIR, - DISABLE_CHECK_QUEUE, INPUTS_DIR, NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, - TARGET_WORKERS, UNIQUE_REPORTS_DIR, + add_cmd_options, CmdType, COVERAGE_DIR, CRASHES_DIR, DISABLE_CHECK_QUEUE, INPUTS_DIR, + NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, UNIQUE_REPORTS_DIR, }, - libfuzzer_coverage::build_coverage_config, - libfuzzer_crash_report::build_report_config, - libfuzzer_fuzz::build_fuzz_config, + libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args}, + libfuzzer_crash_report::{build_report_config, build_shared_args as build_crash_args}, + libfuzzer_fuzz::{build_fuzz_config, build_shared_args as build_fuzz_args}, }, tasks::{ coverage::libfuzzer_coverage::CoverageTask, fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask, @@ -18,7 +17,8 @@ use crate::{ }, }; use anyhow::Result; -use clap::{App, Arg, SubCommand}; +use clap::{App, SubCommand}; +use std::collections::HashSet; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let fuzz_config = build_fuzz_config(args)?; @@ -45,52 +45,20 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { pub fn args(name: &'static str) -> App<'static, 'static> { let mut app = SubCommand::with_name(name).about("run a local libfuzzer & crash reporting task"); - app = add_cmd_options(CmdType::Target, true, true, true, app); + let mut used = HashSet::new(); + for args in &[ + build_fuzz_args(), + build_crash_args(), + build_coverage_args(true), + ] { + for arg in args { + if used.contains(arg.b.name) { + continue; + } + used.insert(arg.b.name.to_string()); + app = app.arg(arg); + } + } - app.arg(Arg::with_name(INPUTS_DIR).takes_value(true).required(true)) - .arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) - .arg( - Arg::with_name(TARGET_WORKERS) - .long(TARGET_WORKERS) - .takes_value(true), - ) - .arg( - Arg::with_name(REPORTS_DIR) - .long(REPORTS_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(COVERAGE_DIR) - .long(COVERAGE_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(NO_REPRO_DIR) - .long(NO_REPRO_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(UNIQUE_REPORTS_DIR) - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name(TARGET_TIMEOUT) - .takes_value(true) - .long(TARGET_TIMEOUT), - ) - .arg( - Arg::with_name(CHECK_RETRY_COUNT) - .takes_value(true) - .long(CHECK_RETRY_COUNT) - .default_value("0"), - ) - .arg( - Arg::with_name(DISABLE_CHECK_QUEUE) - .takes_value(false) - .long(DISABLE_CHECK_QUEUE), - ) + app } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index e4f14fa789..5d7b6856b3 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, + COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, }, tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, }; @@ -12,12 +12,12 @@ use anyhow::Result; use clap::{App, Arg, SubCommand}; use std::path::PathBuf; -pub fn build_coverage_config(args: &clap::ArgMatches<'_>, use_inputs: bool) -> Result { +pub fn build_coverage_config(args: &clap::ArgMatches<'_>, local_job: bool) -> Result { let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); let target_env = get_cmd_env(CmdType::Target, args)?; let target_options = get_cmd_arg(CmdType::Target, args); - let readonly_inputs = if use_inputs { + let readonly_inputs = if local_job { vec![value_t!(args, INPUTS_DIR, PathBuf)?.into()] } else { values_t!(args, READONLY_INPUTS, PathBuf)? @@ -49,20 +49,48 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { task.local_run().await } -pub fn args(name: &'static str) -> App<'static, 'static> { - let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer coverage task"); - - app = add_cmd_options(CmdType::Target, true, true, true, app); - - app.arg( - Arg::with_name(COVERAGE_DIR) +pub fn build_shared_args(local_job: bool) -> Vec> { + let mut args = vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) .takes_value(true) .required(true), - ) - .arg( - Arg::with_name(READONLY_INPUTS) + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) .takes_value(true) - .required(true) .multiple(true), - ) + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set last"), + Arg::with_name(COVERAGE_DIR) + .takes_value(true) + .required(!local_job) + .long(COVERAGE_DIR), + ]; + if local_job { + args.push( + Arg::with_name(INPUTS_DIR) + .long(INPUTS_DIR) + .takes_value(true) + .required(true), + ) + } else { + args.push( + Arg::with_name(READONLY_INPUTS) + .takes_value(true) + .required(true) + .long(READONLY_INPUTS) + .multiple(true), + ) + } + args +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("execute a local-only libfuzzer coverage task") + .args(&build_shared_args(false)) } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index f928a8ff17..8d835bccf8 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -4,8 +4,8 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, - TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, + TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::report::libfuzzer_report::{Config, ReportTask}, }; @@ -59,47 +59,53 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { ReportTask::new(config).local_run().await } -pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { - app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) - .arg( - Arg::with_name(REPORTS_DIR) - .long(REPORTS_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(NO_REPRO_DIR) - .long(NO_REPRO_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(UNIQUE_REPORTS_DIR) - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name(TARGET_TIMEOUT) - .takes_value(true) - .long(TARGET_TIMEOUT), - ) - .arg( - Arg::with_name(CHECK_RETRY_COUNT) - .takes_value(true) - .long(CHECK_RETRY_COUNT) - .default_value("0"), - ) - .arg( - Arg::with_name(DISABLE_CHECK_QUEUE) - .takes_value(false) - .long(DISABLE_CHECK_QUEUE), - ) +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set last"), + Arg::with_name(CRASHES_DIR) + .long(CRASHES_DIR) + .takes_value(true) + .required(true), + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) + .takes_value(true) + .required(false), + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) + .takes_value(true) + .required(false), + Arg::with_name(UNIQUE_REPORTS_DIR) + .long(UNIQUE_REPORTS_DIR) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT), + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + Arg::with_name(DISABLE_CHECK_QUEUE) + .takes_value(false) + .long(DISABLE_CHECK_QUEUE), + ] } pub fn args(name: &'static str) -> App<'static, 'static> { - let mut app = - SubCommand::with_name(name).about("execute a local-only libfuzzer crash report task"); - - app = add_cmd_options(CmdType::Target, true, true, true, app); - add_report_options(app) + SubCommand::with_name(name) + .about("execute a local-only libfuzzer crash report task") + .args(&build_shared_args()) } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index 8e11efd2a4..ad8c8e5627 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -4,7 +4,7 @@ use crate::{ local::common::{ add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CRASHES_DIR, INPUTS_DIR, TARGET_WORKERS, + CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, }, tasks::fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, }; @@ -45,15 +45,38 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { LibFuzzerFuzzTask::new(config)?.local_run().await } +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set last"), + Arg::with_name(INPUTS_DIR) + .long(INPUTS_DIR) + .takes_value(true) + .required(true), + Arg::with_name(CRASHES_DIR) + .long(CRASHES_DIR) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_WORKERS) + .long(TARGET_WORKERS) + .takes_value(true), + ] +} + pub fn args(name: &'static str) -> App<'static, 'static> { - let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer fuzzing task"); - - app = add_cmd_options(CmdType::Target, true, true, true, app); - app.arg(Arg::with_name(INPUTS_DIR).takes_value(true).required(true)) - .arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) - .arg( - Arg::with_name(TARGET_WORKERS) - .long(TARGET_WORKERS) - .takes_value(true), - ) + SubCommand::with_name(name) + .about("execute a local-only libfuzzer fuzzing task") + .args(&build_shared_args()) } diff --git a/src/agent/onefuzz-agent/src/local/mod.rs b/src/agent/onefuzz-agent/src/local/mod.rs index 1840ddd492..7bb84bc571 100644 --- a/src/agent/onefuzz-agent/src/local/mod.rs +++ b/src/agent/onefuzz-agent/src/local/mod.rs @@ -9,3 +9,4 @@ pub mod libfuzzer; pub mod libfuzzer_coverage; pub mod libfuzzer_crash_report; pub mod libfuzzer_fuzz; +pub mod radamsa; diff --git a/src/agent/onefuzz-agent/src/local/radamsa.rs b/src/agent/onefuzz-agent/src/local/radamsa.rs new file mode 100644 index 0000000000..f47c4a53b5 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/radamsa.rs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::{ + common::{ + add_cmd_options, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, + DISABLE_CHECK_QUEUE, INPUTS_DIR, NO_REPRO_DIR, READONLY_INPUTS, RENAME_OUTPUT, + REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, TOOLS_DIR, UNIQUE_REPORTS_DIR, + }, + generic_crash_report::build_report_config, + generic_generator::build_fuzz_config, + }, + tasks::{fuzz::generator::GeneratorTask, report::generic::ReportTask}, +}; + +use anyhow::Result; +use clap::{App, Arg, SubCommand}; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let fuzz_config = build_fuzz_config(args)?; + let report_config = build_report_config(args)?; + + let fuzzer = GeneratorTask::new(fuzz_config); + let fuzz_task = fuzzer.local_run(); + + let report = ReportTask::new(&report_config); + let report_task = report.local_run(); + + tokio::try_join!(fuzz_task, report_task)?; + + Ok(()) +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("run a local libfuzzer & crash reporting task"); + + app = add_cmd_options(CmdType::Generator, true, true, true, app); + app = add_cmd_options(CmdType::Target, true, true, true, app); + app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) + .arg( + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name(UNIQUE_REPORTS_DIR) + .takes_value(true) + .required(true), + ) + .arg(Arg::with_name(TOOLS_DIR).takes_value(true).long(TOOLS_DIR)) + .arg( + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + ) + .arg( + Arg::with_name(CHECK_ASAN_LOG) + .takes_value(false) + .long(CHECK_ASAN_LOG), + ) + .arg( + Arg::with_name(RENAME_OUTPUT) + .takes_value(false) + .long(RENAME_OUTPUT), + ) + .arg( + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT) + .default_value("30"), + ) + .arg( + Arg::with_name("disable_check_debugger") + .takes_value(false) + .long("disable_check_debugger"), + ) + .arg( + Arg::with_name(READONLY_INPUTS) + .takes_value(true) + .required(true) + .multiple(true), + ) +} diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index bcf4f4364c..7bcece53b2 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -67,7 +67,7 @@ impl<'a> ReportTask<'a> { Self { config, poller } } - pub async fn local_run(&mut self) -> Result<()> { + pub async fn local_run(&self) -> Result<()> { let processor = GenericReportProcessor::new(&self.config, None); info!("Starting generic crash report task"); From 557f11d78cd7abeecad7ed8c0a29e925db48d0dc Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 24 Dec 2020 00:04:48 -0500 Subject: [PATCH 12/30] continued dev --- src/agent/onefuzz-agent/src/local/common.rs | 2 +- src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs | 2 +- src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs | 2 +- src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index d2c89d33ab..a6a2879cfc 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -2,7 +2,7 @@ use crate::tasks::config::CommonConfig; use crate::tasks::utils::parse_key_value; use anyhow::Result; use clap::{App, Arg, ArgMatches}; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap}; use uuid::Uuid; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 5d7b6856b3..44f140500f 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -3,7 +3,7 @@ use crate::{ local::common::{ - add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, }, tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index 8d835bccf8..9c54094443 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -3,7 +3,7 @@ use crate::{ local::common::{ - add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index ad8c8e5627..91fe5eb825 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -3,7 +3,7 @@ use crate::{ local::common::{ - add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, }, tasks::fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, From e790aa15e0c3afbadbbd45e54cb71ea070e8a1d5 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 24 Dec 2020 00:24:04 -0500 Subject: [PATCH 13/30] continued dev --- .../src/local/generic_crash_report.rs | 109 +++++++++--------- .../src/local/generic_generator.rs | 109 +++++++++++------- .../onefuzz-agent/src/local/libfuzzer.rs | 5 +- src/agent/onefuzz-agent/src/local/radamsa.rs | 83 +++---------- 4 files changed, 142 insertions(+), 164 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 88ab63fe4f..0afdeeb83e 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -3,9 +3,9 @@ use crate::{ local::common::{ - add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, - REPORTS_DIR, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_ASAN_LOG, + CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, + TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::report::generic::{Config, ReportTask}, }; @@ -65,56 +65,59 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { ReportTask::new(&config).local_run().await } -pub fn add_report_options(app: App<'static, 'static>) -> App<'static, 'static> { - app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) - .arg( - Arg::with_name(REPORTS_DIR) - .long(REPORTS_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(NO_REPRO_DIR) - .long(NO_REPRO_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(UNIQUE_REPORTS_DIR) - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name(TARGET_TIMEOUT) - .takes_value(true) - .long(TARGET_TIMEOUT) - .default_value("30"), - ) - .arg( - Arg::with_name(CHECK_RETRY_COUNT) - .takes_value(true) - .long(CHECK_RETRY_COUNT) - .default_value("0"), - ) - .arg( - Arg::with_name(DISABLE_CHECK_QUEUE) - .takes_value(false) - .long(DISABLE_CHECK_QUEUE), - ) - .arg( - Arg::with_name(CHECK_ASAN_LOG) - .takes_value(false) - .long(CHECK_ASAN_LOG), - ) - .arg( - Arg::with_name("disable_check_debugger") - .takes_value(false) - .long("disable_check_debugger"), - ) +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set last"), + Arg::with_name(CRASHES_DIR) + .long(CRASHES_DIR) + .takes_value(true) + .required(true), + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) + .takes_value(true) + .required(false), + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) + .takes_value(true) + .required(false), + Arg::with_name(UNIQUE_REPORTS_DIR) + .long(UNIQUE_REPORTS_DIR) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT) + .default_value("30"), + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + Arg::with_name(DISABLE_CHECK_QUEUE) + .takes_value(false) + .long(DISABLE_CHECK_QUEUE), + Arg::with_name(CHECK_ASAN_LOG) + .takes_value(false) + .long(CHECK_ASAN_LOG), + Arg::with_name("disable_check_debugger") + .takes_value(false) + .long("disable_check_debugger"), + ] } - pub fn args(name: &'static str) -> App<'static, 'static> { - let mut app = SubCommand::with_name(name).about("execute a local-only generic crash report"); - app = add_cmd_options(CmdType::Target, true, true, true, app); - add_report_options(app) + SubCommand::with_name(name) + .about("execute a local-only generic crash report") + .args(&build_shared_args()) } diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index d60770c96f..36a84793c3 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -3,9 +3,10 @@ use crate::{ local::common::{ - add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, READONLY_INPUTS, RENAME_OUTPUT, - TARGET_TIMEOUT, TOOLS_DIR, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_ASAN_LOG, + CHECK_RETRY_COUNT, CRASHES_DIR, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, + READONLY_INPUTS, RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, + TOOLS_DIR, }, tasks::fuzz::generator::{Config, GeneratorTask}, }; @@ -69,44 +70,68 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { GeneratorTask::new(config).local_run().await } -pub fn args(name: &'static str) -> App<'static, 'static> { - let mut app = SubCommand::with_name(name).about("execute a local-only generator fuzzing task"); +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set last"), + Arg::with_name(GENERATOR_EXE) + .long(GENERATOR_EXE) + .takes_value(true) + .required(true), + Arg::with_name(GENERATOR_ENV) + .long(GENERATOR_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(GENERATOR_OPTIONS) + .long(GENERATOR_OPTIONS) + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set last"), + Arg::with_name(CRASHES_DIR) + .takes_value(true) + .required(true) + .long(CRASHES_DIR), + Arg::with_name(READONLY_INPUTS) + .takes_value(true) + .required(true) + .multiple(true) + .long(READONLY_INPUTS), + Arg::with_name(TOOLS_DIR).takes_value(true).long(TOOLS_DIR), + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + Arg::with_name(CHECK_ASAN_LOG) + .takes_value(false) + .long(CHECK_ASAN_LOG), + Arg::with_name(RENAME_OUTPUT) + .takes_value(false) + .long(RENAME_OUTPUT), + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT) + .default_value("30"), + Arg::with_name("disable_check_debugger") + .takes_value(false) + .long("disable_check_debugger"), + ] +} - app = add_cmd_options(CmdType::Generator, true, true, true, app); - app = add_cmd_options(CmdType::Target, true, true, true, app); - app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) - .arg( - Arg::with_name(READONLY_INPUTS) - .takes_value(true) - .required(true) - .multiple(true), - ) - .arg(Arg::with_name(TOOLS_DIR).takes_value(true)) - .arg( - Arg::with_name(CHECK_RETRY_COUNT) - .takes_value(true) - .long(CHECK_RETRY_COUNT) - .default_value("0"), - ) - .arg( - Arg::with_name(CHECK_ASAN_LOG) - .takes_value(false) - .long(CHECK_ASAN_LOG), - ) - .arg( - Arg::with_name(RENAME_OUTPUT) - .takes_value(false) - .long(RENAME_OUTPUT), - ) - .arg( - Arg::with_name(TARGET_TIMEOUT) - .takes_value(true) - .long(TARGET_TIMEOUT) - .default_value("30"), - ) - .arg( - Arg::with_name("disable_check_debugger") - .takes_value(false) - .long("disable_check_debugger"), - ) +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("execute a local-only generator fuzzing task") + .args(&build_shared_args()) } diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index 6fff64824f..9470204cf9 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -3,10 +3,7 @@ use crate::{ local::{ - common::{ - add_cmd_options, CmdType, COVERAGE_DIR, CRASHES_DIR, DISABLE_CHECK_QUEUE, INPUTS_DIR, - NO_REPRO_DIR, REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, UNIQUE_REPORTS_DIR, - }, + common::COVERAGE_DIR, libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args}, libfuzzer_crash_report::{build_report_config, build_shared_args as build_crash_args}, libfuzzer_fuzz::{build_fuzz_config, build_shared_args as build_fuzz_args}, diff --git a/src/agent/onefuzz-agent/src/local/radamsa.rs b/src/agent/onefuzz-agent/src/local/radamsa.rs index f47c4a53b5..7d0ba277c5 100644 --- a/src/agent/onefuzz-agent/src/local/radamsa.rs +++ b/src/agent/onefuzz-agent/src/local/radamsa.rs @@ -3,19 +3,14 @@ use crate::{ local::{ - common::{ - add_cmd_options, CmdType, CHECK_ASAN_LOG, CHECK_RETRY_COUNT, CRASHES_DIR, - DISABLE_CHECK_QUEUE, INPUTS_DIR, NO_REPRO_DIR, READONLY_INPUTS, RENAME_OUTPUT, - REPORTS_DIR, TARGET_TIMEOUT, TARGET_WORKERS, TOOLS_DIR, UNIQUE_REPORTS_DIR, - }, - generic_crash_report::build_report_config, - generic_generator::build_fuzz_config, + generic_crash_report::{build_report_config, build_shared_args as build_crash_args}, + generic_generator::{build_fuzz_config, build_shared_args as build_fuzz_args}, }, tasks::{fuzz::generator::GeneratorTask, report::generic::ReportTask}, }; - use anyhow::Result; -use clap::{App, Arg, SubCommand}; +use clap::{App, SubCommand}; +use std::collections::HashSet; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let fuzz_config = build_fuzz_config(args)?; @@ -33,60 +28,18 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { } pub fn args(name: &'static str) -> App<'static, 'static> { - let mut app = SubCommand::with_name(name).about("run a local libfuzzer & crash reporting task"); - - app = add_cmd_options(CmdType::Generator, true, true, true, app); - app = add_cmd_options(CmdType::Target, true, true, true, app); - app.arg(Arg::with_name(CRASHES_DIR).takes_value(true).required(true)) - .arg( - Arg::with_name(REPORTS_DIR) - .long(REPORTS_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(NO_REPRO_DIR) - .long(NO_REPRO_DIR) - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(UNIQUE_REPORTS_DIR) - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name(TOOLS_DIR).takes_value(true).long(TOOLS_DIR)) - .arg( - Arg::with_name(CHECK_RETRY_COUNT) - .takes_value(true) - .long(CHECK_RETRY_COUNT) - .default_value("0"), - ) - .arg( - Arg::with_name(CHECK_ASAN_LOG) - .takes_value(false) - .long(CHECK_ASAN_LOG), - ) - .arg( - Arg::with_name(RENAME_OUTPUT) - .takes_value(false) - .long(RENAME_OUTPUT), - ) - .arg( - Arg::with_name(TARGET_TIMEOUT) - .takes_value(true) - .long(TARGET_TIMEOUT) - .default_value("30"), - ) - .arg( - Arg::with_name("disable_check_debugger") - .takes_value(false) - .long("disable_check_debugger"), - ) - .arg( - Arg::with_name(READONLY_INPUTS) - .takes_value(true) - .required(true) - .multiple(true), - ) + let mut app = SubCommand::with_name(name).about("run a local generator & crash reporting job"); + + let mut used = HashSet::new(); + for args in &[build_fuzz_args(), build_crash_args()] { + for arg in args { + if used.contains(arg.b.name) { + continue; + } + used.insert(arg.b.name.to_string()); + app = app.arg(arg); + } + } + + app } From 8a52e5470f012038329407ec14a6f9fe9a77719f Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 24 Dec 2020 00:26:27 -0500 Subject: [PATCH 14/30] fmt --- src/agent/onefuzz-agent/src/local/common.rs | 2 +- src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs | 4 ++-- src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs | 6 +++--- src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index a6a2879cfc..493144b9fd 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -2,7 +2,7 @@ use crate::tasks::config::CommonConfig; use crate::tasks::utils::parse_key_value; use anyhow::Result; use clap::{App, Arg, ArgMatches}; -use std::collections::{HashMap}; +use std::collections::HashMap; use uuid::Uuid; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 44f140500f..92030deef5 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -3,8 +3,8 @@ use crate::{ local::common::{ - build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, COVERAGE_DIR, + INPUTS_DIR, READONLY_INPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, }, tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, }; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index 9c54094443..c24dac646d 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -3,9 +3,9 @@ use crate::{ local::common::{ - build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, - TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_RETRY_COUNT, + CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, TARGET_EXE, + TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, }, tasks::report::libfuzzer_report::{Config, ReportTask}, }; diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index 91fe5eb825..2718d29113 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -3,8 +3,8 @@ use crate::{ local::common::{ - build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, - CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CRASHES_DIR, + INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, }, tasks::fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, }; From d8d3d69682b795e19bf8d2582ef71a2ab9fac5a8 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 5 Jan 2021 09:03:43 -0500 Subject: [PATCH 15/30] make continuous monitor/sync NOOPs without URLs --- src/agent/onefuzz/src/syncdir.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index c87cea1b21..6845e642cb 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -85,7 +85,8 @@ impl SyncedDir { delay_seconds: Option, ) -> Result<()> { if self.url.is_none() { - bail!("only able to sync with remote URLs"); + verbose!("not continuously syncing, as SyncDir does not have a remote URL"); + return Ok(()); } let delay_seconds = delay_seconds.unwrap_or(DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS); @@ -139,7 +140,8 @@ impl SyncedDir { /// a directory, and may reset it. pub async fn monitor_results(&self, event: Event) -> Result<()> { if self.url.is_none() { - bail!("only able to monitor with remote URLs"); + verbose!("not monitoring for results, as SyncDir does not have a remote URL"); + return Ok(()); } loop { From 7096dc1b796457c9bda4b2dacd15129a9d93d42a Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 5 Jan 2021 14:13:10 -0500 Subject: [PATCH 16/30] address feedback --- src/agent/onefuzz-agent/src/local/common.rs | 5 +- .../src/local/generic_crash_report.rs | 6 +- .../src/local/generic_generator.rs | 17 +-- .../src/local/libfuzzer_coverage.rs | 5 +- .../src/local/libfuzzer_crash_report.rs | 5 +- .../onefuzz-agent/src/local/libfuzzer_fuzz.rs | 5 +- src/agent/onefuzz-agent/src/local/radamsa.rs | 2 +- src/agent/onefuzz-agent/src/tasks/config.rs | 4 +- .../onefuzz-agent/src/tasks/fuzz/generator.rs | 19 +--- .../src/tasks/report/crash_report.rs | 103 +++++++----------- .../onefuzz-agent/src/tasks/report/generic.rs | 23 +--- .../src/tasks/report/libfuzzer_report.rs | 24 +--- src/agent/onefuzz/src/syncdir.rs | 48 ++++---- 13 files changed, 106 insertions(+), 160 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index 493144b9fd..5275b37d97 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -66,9 +66,8 @@ pub fn add_cmd_options( Arg::with_name(arg_name) .long(arg_name) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set env first"), + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), ) } app diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 0afdeeb83e..7c4680c571 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -76,11 +76,11 @@ pub fn build_shared_args() -> Vec> { .takes_value(true) .multiple(true), Arg::with_name(TARGET_OPTIONS) + .default_value("{input}") .long(TARGET_OPTIONS) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set last"), + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), Arg::with_name(CRASHES_DIR) .long(CRASHES_DIR) .takes_value(true) diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs index 36a84793c3..e452078660 100644 --- a/src/agent/onefuzz-agent/src/local/generic_generator.rs +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -31,7 +31,7 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { let rename_output = args.is_present(RENAME_OUTPUT); let check_asan_log = args.is_present(CHECK_ASAN_LOG); - let check_debugger = args.is_present("disable_check_debugger"); + let check_debugger = !args.is_present("disable_check_debugger"); let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; let target_timeout = Some(value_t!(args, TARGET_TIMEOUT, u64)?); @@ -67,7 +67,7 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let config = build_fuzz_config(args)?; - GeneratorTask::new(config).local_run().await + GeneratorTask::new(config).run().await } pub fn build_shared_args() -> Vec> { @@ -81,13 +81,14 @@ pub fn build_shared_args() -> Vec> { .takes_value(true) .multiple(true), Arg::with_name(TARGET_OPTIONS) + .default_value("{input}") .long(TARGET_OPTIONS) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set last"), + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), Arg::with_name(GENERATOR_EXE) .long(GENERATOR_EXE) + .default_value("radamsa") .takes_value(true) .required(true), Arg::with_name(GENERATOR_ENV) @@ -97,9 +98,9 @@ pub fn build_shared_args() -> Vec> { Arg::with_name(GENERATOR_OPTIONS) .long(GENERATOR_OPTIONS) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set last"), + .value_delimiter(" ") + .default_value("-H sha256 -o {generated_inputs}/input-%h.%s -n 100 -r {input_corpus}") + .help("Use a quoted string with space separation to denote multiple arguments"), Arg::with_name(CRASHES_DIR) .takes_value(true) .required(true) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs index 92030deef5..e451d68a32 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -62,9 +62,8 @@ pub fn build_shared_args(local_job: bool) -> Vec> { Arg::with_name(TARGET_OPTIONS) .long(TARGET_OPTIONS) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set last"), + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), Arg::with_name(COVERAGE_DIR) .takes_value(true) .required(!local_job) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index c24dac646d..6c5865c257 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -72,9 +72,8 @@ pub fn build_shared_args() -> Vec> { Arg::with_name(TARGET_OPTIONS) .long(TARGET_OPTIONS) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set last"), + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), Arg::with_name(CRASHES_DIR) .long(CRASHES_DIR) .takes_value(true) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index 2718d29113..891a82d4af 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -58,9 +58,8 @@ pub fn build_shared_args() -> Vec> { Arg::with_name(TARGET_OPTIONS) .long(TARGET_OPTIONS) .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set last"), + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), Arg::with_name(INPUTS_DIR) .long(INPUTS_DIR) .takes_value(true) diff --git a/src/agent/onefuzz-agent/src/local/radamsa.rs b/src/agent/onefuzz-agent/src/local/radamsa.rs index 7d0ba277c5..f10c23f848 100644 --- a/src/agent/onefuzz-agent/src/local/radamsa.rs +++ b/src/agent/onefuzz-agent/src/local/radamsa.rs @@ -17,7 +17,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let report_config = build_report_config(args)?; let fuzzer = GeneratorTask::new(fuzz_config); - let fuzz_task = fuzzer.local_run(); + let fuzz_task = fuzzer.run(); let report = ReportTask::new(&report_config); let report_task = report.local_run(); diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 15ba395b07..db4a2b702f 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -157,9 +157,7 @@ impl Config { Config::LibFuzzerMerge(config) => merge::libfuzzer_merge::spawn(Arc::new(config)).await, Config::GenericAnalysis(config) => analysis::generic::spawn(config).await, Config::GenericGenerator(config) => { - fuzz::generator::GeneratorTask::new(config) - .managed_run() - .await + fuzz::generator::GeneratorTask::new(config).run().await } Config::GenericSupervisor(config) => fuzz::supervisor::spawn(config).await, Config::GenericMerge(config) => merge::generic::spawn(Arc::new(config)).await, diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index f1b21ffb1f..c837a533f5 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -61,20 +61,13 @@ impl GeneratorTask { Self { config } } - pub async fn local_run(&self) -> Result<()> { - self.config.crashes.init().await?; - for dir in &self.config.readonly_inputs { - dir.init().await?; - } - - self.fuzzing_loop(None).await - } - - pub async fn managed_run(&self) -> Result<()> { + pub async fn run(&self) -> Result<()> { self.config.crashes.init().await?; if let Some(tools) = &self.config.tools { - tools.init_pull().await?; - set_executable(&tools.path).await?; + if tools.url.is_some() { + tools.init_pull().await?; + set_executable(&tools.path).await?; + } } let hb_client = self.config.common.init_heartbeat().await?; @@ -143,8 +136,8 @@ impl GeneratorTask { let destination_file = self.config.crashes.path.join(destination_file); if tester.is_crash(file.path()).await? { - info!("Crash found, path = {}", file.path().display()); fs::rename(file.path(), &destination_file).await?; + verbose!("crash found {}", destination_file.display()); } } Ok(()) diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index d2e8dd10df..bbcd422b6d 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -4,12 +4,15 @@ use anyhow::Result; use onefuzz::{ asan::AsanLog, - blob::{BlobClient, BlobContainerUrl, BlobUrl}, + blob::{BlobClient, BlobUrl}, fs::exists, syncdir::SyncedDir, - telemetry::Event::{new_report, new_unable_to_reproduce, new_unique_report}, + telemetry::{ + Event::{new_report, new_unable_to_reproduce, new_unique_report}, + EventData, + }, }; -use reqwest::StatusCode; +use reqwest::{StatusCode, Url}; use reqwest_retry::SendRetry; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -62,43 +65,43 @@ pub enum CrashTestResult { } // Conditionally upload a report, if it would not be a duplicate. -// -// Use SHA-256 of call stack as dedupe key. -async fn upload_deduped(report: &CrashReport, container: &BlobContainerUrl) -> Result<()> { +async fn upload(report: &T, url: Url) -> Result { let blob = BlobClient::new(); - let deduped_name = report.unique_blob_name(); - let deduped_url = container.blob(deduped_name).url(); let result = blob - .put(deduped_url) + .put(url) .json(report) // Conditional PUT, only if-not-exists. .header("If-None-Match", "*") .send_retry_default() .await?; - if result.status() != StatusCode::NOT_MODIFIED { - event!(new_unique_report;); - } - Ok(()) -} - -async fn upload_report(report: &CrashReport, container: &BlobContainerUrl) -> Result<()> { - event!(new_report;); - let blob = BlobClient::new(); - let url = container.blob(report.blob_name()).url(); - blob.put(url).json(report).send_retry_default().await?; - Ok(()) + Ok(result.status() != StatusCode::NOT_MODIFIED) } -async fn upload_no_repro(report: &NoCrash, container: &BlobContainerUrl) -> Result<()> { - event!(new_unable_to_reproduce;); - let blob = BlobClient::new(); - let url = container.blob(report.blob_name()).url(); - blob.put(url).json(report).send_retry_default().await?; - Ok(()) +async fn upload_or_save_local( + report: &T, + dest_name: &str, + container: &SyncedDir, +) -> Result { + match &container.url { + Some(blob_url) => { + let url = blob_url.blob(dest_name).url(); + upload(report, url).await + } + None => { + let path = container.path.join(dest_name); + if !exists(&path).await? { + let data = serde_json::to_vec(&report)?; + fs::write(path, data).await?; + Ok(true) + } else { + Ok(false) + } + } + } } impl CrashTestResult { - pub async fn save_local( + pub async fn save( &self, unique_reports: &SyncedDir, reports: &Option, @@ -106,48 +109,26 @@ impl CrashTestResult { ) -> Result<()> { match self { Self::CrashReport(report) => { - let unique_path = unique_reports.path.join(report.blob_name()); - let data = serde_json::to_vec(&report)?; - if !exists(&unique_path).await? { - fs::write(unique_path, &data).await?; + // Use SHA-256 of call stack as dedupe key. + let name = report.unique_blob_name(); + if upload_or_save_local(&report, &name, unique_reports).await? { + event!(new_unique_report; EventData::Path = name); } if let Some(reports) = reports { - let report_path = reports.path.join(report.blob_name()); - if !exists(&report_path).await? { - fs::write(report_path, &data).await?; - } - } - } - Self::NoRepro(report) => { - if let Some(no_repro) = no_repro { - let no_repro_path = no_repro.path.join(report.blob_name()); - if !exists(&no_repro_path).await? { - let data = serde_json::to_vec(&report)?; - fs::write(no_repro_path, &data).await?; + let name = report.blob_name(); + if upload_or_save_local(&report, &name, reports).await? { + event!(new_report; EventData::Path = name); } } } - } - Ok(()) - } - pub async fn upload( - &self, - unique_reports: &SyncedDir, - reports: &Option, - no_repro: &Option, - ) -> Result<()> { - match self { - Self::CrashReport(report) => { - upload_deduped(report, unique_reports.url()?).await?; - if let Some(reports) = reports { - upload_report(report, reports.url()?).await?; - } - } Self::NoRepro(report) => { if let Some(no_repro) = no_repro { - upload_no_repro(report, no_repro.url()?).await?; + let name = report.blob_name(); + if upload_or_save_local(&report, &name, no_repro).await? { + event!(new_unable_to_reproduce; EventData::Path = name); + } } } } diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index 7bcece53b2..c977cf6036 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -68,7 +68,7 @@ impl<'a> ReportTask<'a> { } pub async fn local_run(&self) -> Result<()> { - let processor = GenericReportProcessor::new(&self.config, None); + let mut processor = GenericReportProcessor::new(&self.config, None); info!("Starting generic crash report task"); let crashes = match &self.config.crashes { @@ -78,14 +78,14 @@ impl<'a> ReportTask<'a> { let mut read_dir = tokio::fs::read_dir(&crashes.path).await?; while let Some(crash) = read_dir.next().await { - processor.test_local(crash?.path()).await?; + processor.process(None, &crash?.path()).await?; } if self.config.check_queue { let mut monitor = DirectoryMonitor::new(&crashes.path); monitor.start()?; while let Some(crash) = monitor.next().await { - processor.test_local(crash).await?; + processor.process(None, &crash).await?; } } Ok(()) @@ -136,20 +136,6 @@ impl<'a> GenericReportProcessor<'a> { } } - async fn test_local(&self, input: PathBuf) -> Result<()> { - info!("generating crash report for: {}", input.display()); - let result = self.test_input(None, &input).await?; - result - .save_local( - &self.config.unique_reports, - &self.config.reports, - &self.config.no_repro, - ) - .await?; - - Ok(()) - } - pub async fn test_input( &self, input_url: Option, @@ -213,9 +199,10 @@ impl<'a> GenericReportProcessor<'a> { #[async_trait] impl<'a> Processor for GenericReportProcessor<'a> { async fn process(&mut self, url: Option, input: &Path) -> Result<()> { + verbose!("generating crash report for: {}", input.display()); let report = self.test_input(url, input).await?; report - .upload( + .save( &self.config.unique_reports, &self.config.reports, &self.config.no_repro, diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index e43da506a4..d8dd499106 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -58,7 +58,7 @@ impl ReportTask { } pub async fn local_run(&self) -> Result<()> { - let processor = AsanProcessor::new(self.config.clone()).await?; + let mut processor = AsanProcessor::new(self.config.clone()).await?; let crashes = match &self.config.crashes { Some(x) => x, None => bail!("missing crashes directory"), @@ -74,14 +74,14 @@ impl ReportTask { let mut read_dir = tokio::fs::read_dir(&crashes.path).await?; while let Some(crash) = read_dir.next().await { - processor.test_local(crash?.path()).await?; + processor.process(None, &crash?.path()).await?; } if self.config.check_queue { let mut monitor = DirectoryMonitor::new(crashes.path.clone()); monitor.start()?; while let Some(crash) = monitor.next().await { - processor.test_local(crash).await?; + processor.process(None, &crash).await?; } } @@ -121,20 +121,6 @@ impl AsanProcessor { }) } - async fn test_local(&self, input: PathBuf) -> Result<()> { - info!("generating crash report for: {}", input.display()); - let result = self.test_input(None, &input).await?; - result - .save_local( - &self.config.unique_reports, - &self.config.reports, - &self.config.no_repro, - ) - .await?; - - Ok(()) - } - pub async fn test_input( &self, input_url: Option, @@ -195,10 +181,10 @@ impl AsanProcessor { #[async_trait] impl Processor for AsanProcessor { async fn process(&mut self, url: Option, input: &Path) -> Result<()> { - info!("processing libfuzzer crash url:{:?} path:{:?}", url, input); + verbose!("processing libfuzzer crash url:{:?} path:{:?}", url, input); let report = self.test_input(url, input).await?; report - .upload( + .save( &self.config.unique_reports, &self.config.reports, &self.config.no_repro, diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index 6845e642cb..a01075968e 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -32,7 +32,8 @@ pub struct SyncedDir { impl SyncedDir { pub async fn sync(&self, operation: SyncOperation, delete_dst: bool) -> Result<()> { if self.url.is_none() { - bail!("only able to sync with remote URLs"); + verbose!("not syncing as SyncedDir is missing remote URL"); + return Ok(()); } let dir = &self.path; @@ -101,27 +102,24 @@ impl SyncedDir { } } - async fn file_uploader_monitor(&self, event: Event) -> Result<()> { - if self.url.is_none() { - bail!("only able to monitor with remote URLs"); - } - - let url = self.url.as_ref().unwrap().url(); + async fn file_monitor_event(&self, event: Event) -> Result<()> { verbose!("monitoring {}", self.path.display()); - let mut monitor = DirectoryMonitor::new(self.path.clone()); monitor.start()?; - let mut uploader = BlobUploader::new(url); + + let mut uploader = self.url.as_ref().map(|x| BlobUploader::new(x.url())); while let Some(item) = monitor.next().await { event!(event.clone(); EventData::Path = item.display().to_string()); - if let Err(err) = uploader.upload(item.clone()).await { - bail!( - "Couldn't upload file. path:{} dir:{} err:{}", - item.display(), - self.path.display(), - err - ); + if let Some(uploader) = &mut uploader { + if let Err(err) = uploader.upload(item.clone()).await { + bail!( + "Couldn't upload file. path:{} dir:{} err:{}", + item.display(), + self.path.display(), + err + ); + } } } @@ -139,11 +137,6 @@ impl SyncedDir { /// to be initialized, but a user-supplied binary, (such as AFL) logically owns /// a directory, and may reset it. pub async fn monitor_results(&self, event: Event) -> Result<()> { - if self.url.is_none() { - verbose!("not monitoring for results, as SyncDir does not have a remote URL"); - return Ok(()); - } - loop { verbose!("waiting to monitor {}", self.path.display()); @@ -153,7 +146,7 @@ impl SyncedDir { } verbose!("starting monitor for {}", self.path.display()); - self.file_uploader_monitor(event.clone()).await?; + self.file_monitor_event(event.clone()).await?; } } } @@ -169,6 +162,17 @@ pub async fn continuous_sync( operation: SyncOperation, delay_seconds: Option, ) -> Result<()> { + let mut should_loop = false; + for dir in dirs { + if dir.url.is_some() { + should_loop = true; + } + } + if !should_loop { + verbose!("not syncing as SyncDirs do not have remote URLs"); + return Ok(()); + } + let delay_seconds = delay_seconds.unwrap_or(DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS); if delay_seconds == 0 { return Ok(()); From 597e627f955f2fb8b16ccecd99f0732017bec156 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 5 Jan 2021 14:13:27 -0500 Subject: [PATCH 17/30] log telemetry events locally --- src/agent/onefuzz/src/telemetry.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/agent/onefuzz/src/telemetry.rs b/src/agent/onefuzz/src/telemetry.rs index 8f7874c25a..6d9461885a 100644 --- a/src/agent/onefuzz/src/telemetry.rs +++ b/src/agent/onefuzz/src/telemetry.rs @@ -309,6 +309,16 @@ pub fn set_property(entry: EventData) { } } +fn local_log_event(event: &Event, properties: &[EventData]) { + let as_values = properties + .iter() + .map(|x| x.as_values()) + .map(|(x, y)| format!("{}:{}", x, y)) + .collect::>() + .join(" "); + log::log!(log::Level::Info, "{} {}", event.as_str(), as_values); +} + pub fn track_event(event: Event, properties: Vec) { use appinsights::telemetry::Telemetry; @@ -334,6 +344,7 @@ pub fn track_event(event: Event, properties: Vec) { } client.track(evt); } + local_log_event(&event, &properties); } #[macro_export] From 7fc47eab7ce366999a77d2bce1c73405c772339d Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 5 Jan 2021 15:48:18 -0500 Subject: [PATCH 18/30] fix radamsa tests --- src/agent/onefuzz-agent/src/tasks/config.rs | 2 +- .../onefuzz-agent/src/tasks/fuzz/generator.rs | 77 +++++++++++++------ .../src/tasks/generic/input_poller/tests.rs | 2 +- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index db4a2b702f..f28620b64d 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -20,7 +20,7 @@ pub enum ContainerType { Inputs, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Clone, Default)] pub struct CommonConfig { pub job_id: Uuid, diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index 81187b68c3..76df9355ad 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -188,19 +188,27 @@ impl GeneratorTask { } mod tests { + use anyhow::Result; + #[tokio::test] #[cfg(target_os = "linux")] #[ignore] - async fn test_radamsa_linux() { - use super::*; + async fn test_radamsa_linux() -> Result<()> { + use super::{Config, GeneratorTask}; + use crate::tasks::config::CommonConfig; + use onefuzz::syncdir::SyncedDir; + use std::collections::HashMap; use std::env; + use std::path::Path; + use tempfile::tempdir; + + let crashes_temp = tempfile::tempdir()?; + let crashes = crashes_temp.path(); - let radamsa_path = env::var("ONEFUZZ_TEST_RADAMSA_LINUX").unwrap(); - let corpus_dir_temp = tempfile::tempdir().unwrap(); - let corpus_dir = corpus_dir_temp.into_path(); - let seed_file_name = corpus_dir.clone().join("seed.txt"); - let radamsa_output_temp = tempfile::tempdir().unwrap(); - let radamsa_output = radamsa_output_temp.into_path(); + let inputs_temp = tempfile::tempdir().unwrap(); + let inputs = inputs_temp.path(); + let input_file = inputs.join("seed.txt"); + tokio::fs::write(input_file, "test").await?; let generator_options: Vec = vec![ "-o", @@ -214,22 +222,45 @@ mod tests { .map(|p| p.to_string()) .collect(); + let radamsa_path = env::var("ONEFUZZ_TEST_RADAMSA_LINUX")?; let radamsa_as_path = Path::new(&radamsa_path); let radamsa_dir = radamsa_as_path.parent().unwrap(); - let radamsa_exe = String::from("{tools_dir}/radamsa"); - let radamsa_env = HashMap::new(); - - tokio::fs::write(seed_file_name, "test").await.unwrap(); - let _output = generate_input( - &radamsa_exe, - &radamsa_env, - &generator_options, - &radamsa_dir, - corpus_dir, - radamsa_output.clone(), - ) - .await; - let generated_outputs = std::fs::read_dir(radamsa_output.clone()).unwrap(); - assert_eq!(generated_outputs.count(), 100, "No crashes generated"); + + let config = Config { + generator_exe: String::from("{tools_dir}/radamsa"), + generator_options, + readonly_inputs: vec![SyncedDir { + path: inputs.to_path_buf(), + url: None, + }], + crashes: SyncedDir { + path: crashes.to_path_buf(), + url: None, + }, + tools: Some(SyncedDir { + path: radamsa_dir.to_path_buf(), + url: None, + }), + target_exe: Default::default(), + target_env: Default::default(), + target_options: Default::default(), + target_timeout: None, + check_asan_log: false, + check_debugger: false, + rename_output: false, + ensemble_sync_delay: None, + generator_env: HashMap::default(), + check_retry_count: 0, + common: CommonConfig::default(), + }; + let task = GeneratorTask::new(config); + + let generated_inputs = tempdir()?; + task.generate_inputs(inputs.to_path_buf(), generated_inputs.path()) + .await?; + + let count = std::fs::read_dir(generated_inputs.path())?.count(); + assert_eq!(count, 100, "No inputs generated"); + Ok(()) } } diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs index 771ae84c9c..16ffaac771 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs @@ -109,7 +109,7 @@ fn url_fixture(msg: Msg) -> Url { Url::parse(&format!("https://azure.com/c/{}", msg)).unwrap() } -fn input_fixture(dir: &Path, msg: Msg) -> PathBuf { +fn _input_fixture(dir: &Path, msg: Msg) -> PathBuf { let name = msg.to_string(); dir.join(name) } From 44a9a3b1322b81a37986c90e22f93eaf6671e050 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 5 Jan 2021 15:56:47 -0500 Subject: [PATCH 19/30] address comments --- src/agent/onefuzz-agent/src/local/libfuzzer.rs | 2 +- src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs | 2 +- src/agent/onefuzz-agent/src/tasks/config.rs | 2 +- src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs | 4 +--- src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs | 7 +------ 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index 9470204cf9..cce26c9929 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -22,7 +22,7 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let report_config = build_report_config(args)?; let fuzzer = LibFuzzerFuzzTask::new(fuzz_config)?; - let fuzz_task = fuzzer.local_run(); + let fuzz_task = fuzzer.run(); let report = ReportTask::new(report_config); let report_task = report.local_run(); diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs index d37c7b2c3e..0e859d23b9 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -48,7 +48,7 @@ pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let config = build_fuzz_config(args)?; - LibFuzzerFuzzTask::new(config)?.local_run().await + LibFuzzerFuzzTask::new(config)?.run().await } pub fn build_shared_args() -> Vec> { diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index f28620b64d..ea2e965e87 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -141,7 +141,7 @@ impl Config { match self { Config::LibFuzzerFuzz(config) => { fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask::new(config)? - .managed_run() + .run() .await } Config::LibFuzzerReport(config) => { diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index 76df9355ad..6b19030a72 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -188,12 +188,10 @@ impl GeneratorTask { } mod tests { - use anyhow::Result; - #[tokio::test] #[cfg(target_os = "linux")] #[ignore] - async fn test_radamsa_linux() -> Result<()> { + async fn test_radamsa_linux() -> anyhow::Result<()> { use super::{Config, GeneratorTask}; use crate::tasks::config::CommonConfig; use onefuzz::syncdir::SyncedDir; diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index 94f6437de8..0612f94d64 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -80,12 +80,7 @@ impl LibFuzzerFuzzTask { } } - pub async fn local_run(&self) -> Result<()> { - self.init_directories().await?; - self.run_fuzzers(None).await - } - - pub async fn managed_run(&self) -> Result<()> { + pub async fn run(&self) -> Result<()> { self.init_directories().await?; let hb_client = self.config.common.init_heartbeat().await?; From 63a45260a5354a9cdd64b3c3e670f5a2a2f66c2b Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 7 Jan 2021 11:25:03 -0500 Subject: [PATCH 20/30] Don't include None and Unset fields (which render to null) in rendered config --- src/api-service/__app__/onefuzzlib/tasks/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api-service/__app__/onefuzzlib/tasks/scheduler.py b/src/api-service/__app__/onefuzzlib/tasks/scheduler.py index a24265a54a..360c9ebf03 100644 --- a/src/api-service/__app__/onefuzzlib/tasks/scheduler.py +++ b/src/api-service/__app__/onefuzzlib/tasks/scheduler.py @@ -157,7 +157,7 @@ def build_work_unit(task: Task) -> Optional[Tuple[BucketConfig, WorkUnit]]: job_id=task_config.job_id, task_id=task_config.task_id, task_type=task_config.task_type, - config=task_config.json(), + config=task_config.json(exclude_none=True, exclude_unset=True), ) bucket_config = BucketConfig( From 7cd4a660c47d0ff69853e3c03b4c62c2247e8b64 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Fri, 8 Jan 2021 11:13:22 -0500 Subject: [PATCH 21/30] address feedback --- .../src/tasks/analysis/generic.rs | 12 ++-- .../src/tasks/generic/input_poller.rs | 13 ++-- .../src/tasks/generic/input_poller/tests.rs | 65 +++++++++++-------- 3 files changed, 54 insertions(+), 36 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs index 888aba9bd3..4223704cf4 100644 --- a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs @@ -66,10 +66,14 @@ async fn run_existing(config: &Config) -> Result<()> { async fn already_checked(config: &Config, input: &BlobUrl) -> Result { let result = if let Some(crashes) = &config.crashes { - // TODO: this should really check if the URL is None then only check the local path. Otherwise check the container info - crashes.url()?.account() == input.account() - && crashes.url()?.container() == input.container() - && crashes.path.join(input.name()).exists() + match crashes.url() { + Ok(url) => { + url.account() == input.account() + && url.container() == input.container() + && crashes.path.join(input.name()).exists() + } + Err(_) => crashes.path.join(input.name()).exists(), + } } else { false }; diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index ddab7d1d42..252e5e4bd0 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -135,11 +135,14 @@ impl InputPoller { pub async fn seen_in_batch(&self, url: &Url) -> Result { let result = if let Some(batch_dir) = &self.batch_dir { if let Ok(blob) = BlobUrl::new(url.clone()) { - // TODO - this should see if we have a URL container and only check that part if we do - // otherwise only check the local path - batch_dir.url()?.account() == blob.account() - && batch_dir.url()?.container() == blob.container() - && batch_dir.path.join(blob.name()).exists() + match batch_dir.url() { + Ok(batch_url) => { + batch_url.account() == blob.account() + && batch_url.container() == blob.container() + && batch_dir.path.join(blob.name()).exists() + } + Err(_) => batch_dir.path.join(blob.name()).exists(), + } } else { false } diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs index 16ffaac771..464e644324 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs @@ -109,7 +109,7 @@ fn url_fixture(msg: Msg) -> Url { Url::parse(&format!("https://azure.com/c/{}", msg)).unwrap() } -fn _input_fixture(dir: &Path, msg: Msg) -> PathBuf { +fn input_fixture(dir: &Path, msg: Msg) -> PathBuf { let name = msg.to_string(); dir.join(name) } @@ -162,43 +162,54 @@ async fn test_polled_none_parse() { assert_eq!(task.state(), &State::Ready); } -// #[tokio::test] -// async fn test_parsed_download() { -// let mut task = fixture(); +#[tokio::test] +async fn test_parsed_download() { + let mut task = fixture(); -// let msg: Msg = 0; -// let url = url_fixture(msg); -// let input = input_fixture(dir.path(), msg); + let dir = Path::new("etc"); + let msg: Msg = 0; + let url = url_fixture(msg); + let input = input_fixture(&dir, msg); -// task.set_state(State::Parsed(msg, url.clone())); + task.set_state(State::Parsed(msg, url.clone())); -// let mut downloader = TestDownloader::default(); + let mut downloader = TestDownloader::default(); -// task.trigger(Event::Download(&mut downloader)) -// .await -// .unwrap(); + task.trigger(Event::Download(&mut downloader)) + .await + .unwrap(); + + match task.state() { + State::Downloaded(got_msg, got_url, got_path) => { + assert_eq!(*got_msg, msg); + assert_eq!(*got_url, url); + assert_eq!(got_path.file_name(), input.file_name()); + } + _ => { + panic!("unexpected state"); + } + } +} -// assert_eq!(task.state(), &State::Downloaded(msg, url.clone(), input)); -// assert_eq!(downloader.downloaded, vec![url]); -// } +#[tokio::test] +async fn test_downloaded_process() { + let mut task = fixture(); -// #[tokio::test] -// async fn test_downloaded_process() { -// let mut task = fixture(); + let dir = Path::new("etc"); -// let msg: Msg = 0; -// let url = url_fixture(msg); -// let input = input_fixture(dir.path(), msg); + let msg: Msg = 0; + let url = url_fixture(msg); + let input = input_fixture(dir, msg); -// task.set_state(State::Downloaded(msg, url.clone(), input.clone())); + task.set_state(State::Downloaded(msg, url.clone(), input.clone())); -// let mut processor = TestProcessor::default(); + let mut processor = TestProcessor::default(); -// task.trigger(Event::Process(&mut processor)).await.unwrap(); + task.trigger(Event::Process(&mut processor)).await.unwrap(); -// assert_eq!(task.state(), &State::Processed(msg)); -// assert_eq!(processor.processed, vec![(Some(url), input)]); -// } + assert_eq!(task.state(), &State::Processed(msg)); + assert_eq!(processor.processed, vec![(Some(url), input)]); +} #[tokio::test] async fn test_processed_finish() { From 933f8a512b5f8f29b67140d00e772126e12838ae Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Fri, 8 Jan 2021 11:14:47 -0500 Subject: [PATCH 22/30] break early --- src/agent/onefuzz/src/syncdir.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index a01075968e..07862362ba 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -166,6 +166,7 @@ pub async fn continuous_sync( for dir in dirs { if dir.url.is_some() { should_loop = true; + break; } } if !should_loop { From 7134c00dab507e3f1cb8e362de1d9419e2f8d550 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Fri, 8 Jan 2021 11:17:19 -0500 Subject: [PATCH 23/30] address feedback --- src/agent/onefuzz-agent/src/tasks/analysis/generic.rs | 2 +- src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs | 4 ++-- src/agent/onefuzz/src/syncdir.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs index 4223704cf4..b278b16b36 100644 --- a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs @@ -66,7 +66,7 @@ async fn run_existing(config: &Config) -> Result<()> { async fn already_checked(config: &Config, input: &BlobUrl) -> Result { let result = if let Some(crashes) = &config.crashes { - match crashes.url() { + match crashes.try_url() { Ok(url) => { url.account() == input.account() && url.container() == input.container() diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index 252e5e4bd0..b73ba61515 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -124,7 +124,7 @@ impl InputPoller { let dir_relative = input_path.strip_prefix(&dir_path)?; dir_relative.display().to_string() }; - let url = to_process.url().map(|x| x.blob(blob_name).url()).ok(); + let url = to_process.try_url().map(|x| x.blob(blob_name).url()).ok(); processor.process(url, &path).await?; } @@ -135,7 +135,7 @@ impl InputPoller { pub async fn seen_in_batch(&self, url: &Url) -> Result { let result = if let Some(batch_dir) = &self.batch_dir { if let Ok(blob) = BlobUrl::new(url.clone()) { - match batch_dir.url() { + match batch_dir.try_url() { Ok(batch_url) => { batch_url.account() == blob.account() && batch_url.container() == blob.container() diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index 07862362ba..33dcb722c2 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -46,7 +46,7 @@ impl SyncedDir { } } - pub fn url<'a>(&'a self) -> Result<&'a BlobContainerUrl> { + pub fn try_url(&self) -> Result<&BlobContainerUrl> { let url = match &self.url { Some(x) => x, None => bail!("missing URL context"), From 726157f8dc65a87e69bf4b2d047d4cfebb26e618 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Fri, 8 Jan 2021 13:06:45 -0500 Subject: [PATCH 24/30] revert change to batch processing and add logging --- .../onefuzz-agent/src/tasks/analysis/generic.rs | 12 ++++-------- .../src/tasks/coverage/libfuzzer_coverage.rs | 3 ++- .../onefuzz-agent/src/tasks/generic/input_poller.rs | 11 +++-------- src/agent/onefuzz-agent/src/tasks/report/generic.rs | 2 ++ 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs index b278b16b36..5342f4778b 100644 --- a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs @@ -66,14 +66,10 @@ async fn run_existing(config: &Config) -> Result<()> { async fn already_checked(config: &Config, input: &BlobUrl) -> Result { let result = if let Some(crashes) = &config.crashes { - match crashes.try_url() { - Ok(url) => { - url.account() == input.account() - && url.container() == input.container() - && crashes.path.join(input.name()).exists() - } - Err(_) => crashes.path.join(input.name()).exists(), - } + let url = crashes.try_url()?; + url.account() == input.account() + && url.container() == input.container() + && crashes.path.join(input.name()).exists() } else { false }; diff --git a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs index b61093a49e..e7382a7441 100644 --- a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs @@ -127,6 +127,7 @@ impl CoverageTask { async fn process(&mut self) -> Result<()> { let mut processor = CoverageProcessor::new(self.config.clone()).await?; + info!("processing initial dataset"); let mut seen_inputs = false; // Update the total with the coverage from each seed corpus. for dir in &self.config.readonly_inputs { @@ -153,7 +154,7 @@ impl CoverageTask { // If a queue has been provided, poll it for new coverage. if let Some(queue) = &self.config.input_queue { - verbose!("polling queue for new coverage"); + info!("polling queue for new coverage"); let callback = CallbackImpl::new(queue.clone(), processor); self.poller.run(callback).await?; } diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index b73ba61515..10fa5a0014 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -135,14 +135,9 @@ impl InputPoller { pub async fn seen_in_batch(&self, url: &Url) -> Result { let result = if let Some(batch_dir) = &self.batch_dir { if let Ok(blob) = BlobUrl::new(url.clone()) { - match batch_dir.try_url() { - Ok(batch_url) => { - batch_url.account() == blob.account() - && batch_url.container() == blob.container() - && batch_dir.path.join(blob.name()).exists() - } - Err(_) => batch_dir.path.join(blob.name()).exists(), - } + batch_dir.try_url()?.account() == blob.account() + && batch_dir.try_url()?.container() == blob.container() + && batch_dir.path.join(blob.name()).exists() } else { false } diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index 4af0b367c8..885136f6fd 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -94,10 +94,12 @@ impl<'a> ReportTask<'a> { let heartbeat_client = self.config.common.init_heartbeat().await?; let mut processor = GenericReportProcessor::new(&self.config, heartbeat_client); + info!("processing existing crashes"); if let Some(crashes) = &self.config.crashes { self.poller.batch_process(&mut processor, &crashes).await?; } + info!("processing from queue"); if self.config.check_queue { if let Some(queue) = &self.config.input_queue { let callback = CallbackImpl::new(queue.clone(), processor); From 56dfdbf62dd2b8075766cd76141a743f76ebad14 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Fri, 8 Jan 2021 14:22:05 -0500 Subject: [PATCH 25/30] add logging --- src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs | 3 ++- src/agent/onefuzz-agent/src/tasks/report/generic.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index 10fa5a0014..7ba6b63975 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -107,7 +107,7 @@ impl InputPoller { if to_process.url.is_some() { to_process.init_pull().await?; } - info!("checking: {}", to_process.path.display()); + info!("batch processing directory: {}", to_process.path.display()); let mut read_dir = fs::read_dir(&to_process.path).await?; while let Some(file) = read_dir.next().await { @@ -157,6 +157,7 @@ impl InputPoller { } pub async fn run(&mut self, mut cb: impl Callback) -> Result<()> { + info!("starting input queue polling"); loop { match self.state() { State::Polled(None) => { diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index 885136f6fd..e8046e2c88 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -99,7 +99,7 @@ impl<'a> ReportTask<'a> { self.poller.batch_process(&mut processor, &crashes).await?; } - info!("processing from queue"); + info!("processing crashes from queue"); if self.config.check_queue { if let Some(queue) = &self.config.input_queue { let callback = CallbackImpl::new(queue.clone(), processor); From 60c3e9e8dcaec30e970b09cf0c78dd1642d415e7 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 13 Jan 2021 10:01:58 -0500 Subject: [PATCH 26/30] update support for setup_dir --- src/agent/onefuzz-agent/src/local/common.rs | 23 ++++++++++++++- src/agent/onefuzz-agent/src/tasks/config.rs | 29 ++++++++++++++----- .../src/tasks/fuzz/libfuzzer_fuzz.rs | 1 - src/agent/onefuzz-supervisor/src/worker.rs | 1 - 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs index f86f0e4702..2fcc292db1 100644 --- a/src/agent/onefuzz-agent/src/local/common.rs +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -2,10 +2,11 @@ use crate::tasks::config::CommonConfig; use crate::tasks::utils::parse_key_value; use anyhow::Result; use clap::{App, Arg, ArgMatches}; -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; use uuid::Uuid; +pub const SETUP_DIR: &str = "setup_dir"; pub const INPUTS_DIR: &str = "inputs_dir"; pub const CRASHES_DIR: &str = "crashes_dir"; pub const TARGET_WORKERS: &str = "target_workers"; @@ -132,6 +133,12 @@ pub fn add_common_config(app: App<'static, 'static>) -> App<'static, 'static> { .takes_value(true) .required(false), ) + .arg( + Arg::with_name("setup_dir") + .long("setup_dir") + .takes_value(true) + .required(false), + ) } fn get_uuid(name: &str, args: &ArgMatches<'_>) -> Result { @@ -147,6 +154,19 @@ pub fn build_common_config(args: &ArgMatches<'_>) -> Result { let task_id = get_uuid("task_id", args)?; let instance_id = get_uuid("instance_id", args)?; + let setup_dir = if args.is_present(SETUP_DIR) { + value_t!(args, SETUP_DIR, PathBuf)? + } else { + if args.is_present(TARGET_EXE) { + value_t!(args, TARGET_EXE, PathBuf)? + .parent() + .map(|x| x.to_path_buf()) + .unwrap_or_default() + } else { + PathBuf::default() + } + }; + let config = CommonConfig { heartbeat_queue: None, instrumentation_key: None, @@ -154,6 +174,7 @@ pub fn build_common_config(args: &ArgMatches<'_>) -> Result { job_id, task_id, instance_id, + setup_dir, }; Ok(config) } diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 1c4885e990..cc9cc584b5 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -10,7 +10,7 @@ use onefuzz::{ }; use reqwest::Url; use serde::{self, Deserialize}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use uuid::Uuid; @@ -81,16 +81,29 @@ pub enum Config { } impl Config { - pub fn from_file(path: impl AsRef, setup_dir: Option>) -> Result { + pub fn from_file(path: PathBuf, setup_dir: PathBuf) -> Result { let json = std::fs::read_to_string(path)?; - let mut json_config: serde_json::Value = serde_json::from_str(&json)?; + let json_config: serde_json::Value = serde_json::from_str(&json)?; + // override the setup_dir in the config file with the parameter value if specified - if let Some(setup_dir) = setup_dir { - json_config["setup_dir"] = - serde_json::Value::String(setup_dir.as_ref().to_string_lossy().into()); - } + let mut config: Self = serde_json::from_value(json_config)?; + config.common_mut().setup_dir = setup_dir; + + Ok(config) + } - Ok(serde_json::from_value(json_config)?) + fn common_mut(&mut self) -> &mut CommonConfig { + match self { + Config::LibFuzzerFuzz(c) => &mut c.common, + Config::LibFuzzerMerge(c) => &mut c.common, + Config::LibFuzzerReport(c) => &mut c.common, + Config::LibFuzzerCoverage(c) => &mut c.common, + Config::GenericAnalysis(c) => &mut c.common, + Config::GenericMerge(c) => &mut c.common, + Config::GenericReport(c) => &mut c.common, + Config::GenericSupervisor(c) => &mut c.common, + Config::GenericGenerator(c) => &mut c.common, + } } pub fn common(&self) -> &CommonConfig { diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index 6066155331..b9f3b10297 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -162,7 +162,6 @@ impl LibFuzzerFuzzTask { &self.config.target_options, &self.config.target_env, &self.config.common.setup_dir, - &self.config.common.setup_dir, ); let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?; diff --git a/src/agent/onefuzz-supervisor/src/worker.rs b/src/agent/onefuzz-supervisor/src/worker.rs index 56024c192c..c3437c61c2 100644 --- a/src/agent/onefuzz-supervisor/src/worker.rs +++ b/src/agent/onefuzz-supervisor/src/worker.rs @@ -227,7 +227,6 @@ impl IWorkerRunner for WorkerRunner { cmd.current_dir(&working_dir); cmd.arg("managed"); cmd.arg("config.json"); - cmd.arg("-s"); cmd.arg(setup_dir); cmd.stderr(Stdio::piped()); cmd.stdout(Stdio::piped()); From 135cdda6cf899496d69f51e00f02c60d041ebc56 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 13 Jan 2021 15:38:08 -0500 Subject: [PATCH 27/30] continued updates --- .../src/local/generic_crash_report.rs | 3 +- .../onefuzz-agent/src/local/libfuzzer.rs | 20 ++++--- src/agent/onefuzz-agent/src/local/radamsa.rs | 14 ++--- src/agent/onefuzz-agent/src/tasks/config.rs | 4 +- .../onefuzz-agent/src/tasks/fuzz/generator.rs | 52 ++++++++++--------- .../src/tasks/fuzz/libfuzzer_fuzz.rs | 2 +- .../onefuzz-agent/src/tasks/report/generic.rs | 8 +-- .../src/tasks/report/libfuzzer_report.rs | 15 ++++-- src/agent/onefuzz/src/syncdir.rs | 2 +- 9 files changed, 69 insertions(+), 51 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 7c4680c571..4ef3d976c0 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -62,7 +62,7 @@ pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let config = build_report_config(args)?; - ReportTask::new(&config).local_run().await + ReportTask::new(config).local_run().await } pub fn build_shared_args() -> Vec> { @@ -116,6 +116,7 @@ pub fn build_shared_args() -> Vec> { .long("disable_check_debugger"), ] } + pub fn args(name: &'static str) -> App<'static, 'static> { SubCommand::with_name(name) .about("execute a local-only generic crash report") diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs index cce26c9929..9bfb273ae9 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -16,24 +16,30 @@ use crate::{ use anyhow::Result; use clap::{App, SubCommand}; use std::collections::HashSet; +use tokio::task::spawn; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let fuzz_config = build_fuzz_config(args)?; - let report_config = build_report_config(args)?; - let fuzzer = LibFuzzerFuzzTask::new(fuzz_config)?; - let fuzz_task = fuzzer.run(); + let fuzz_task = spawn(async move { fuzzer.run().await }); + let report_config = build_report_config(args)?; let report = ReportTask::new(report_config); - let report_task = report.local_run(); + let report_task = spawn(async move { report.local_run().await }); + if args.is_present(COVERAGE_DIR) { let coverage_config = build_coverage_config(args, true)?; let coverage = CoverageTask::new(coverage_config); - let coverage_task = coverage.local_run(); + let coverage_task = spawn(async move { coverage.local_run().await }); - tokio::try_join!(fuzz_task, report_task, coverage_task)?; + let result = tokio::try_join!(fuzz_task, report_task, coverage_task)?; + result.0?; + result.1?; + result.2?; } else { - tokio::try_join!(fuzz_task, report_task)?; + let result = tokio::try_join!(fuzz_task, report_task)?; + result.0?; + result.1?; } Ok(()) diff --git a/src/agent/onefuzz-agent/src/local/radamsa.rs b/src/agent/onefuzz-agent/src/local/radamsa.rs index f10c23f848..e7fda771f8 100644 --- a/src/agent/onefuzz-agent/src/local/radamsa.rs +++ b/src/agent/onefuzz-agent/src/local/radamsa.rs @@ -11,18 +11,20 @@ use crate::{ use anyhow::Result; use clap::{App, SubCommand}; use std::collections::HashSet; +use tokio::task::spawn; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { let fuzz_config = build_fuzz_config(args)?; - let report_config = build_report_config(args)?; - let fuzzer = GeneratorTask::new(fuzz_config); - let fuzz_task = fuzzer.run(); + let fuzz_task = spawn(async move { fuzzer.run().await }); - let report = ReportTask::new(&report_config); - let report_task = report.local_run(); + let report_config = build_report_config(args)?; + let report = ReportTask::new(report_config); + let report_task = spawn(async move { report.local_run().await }); - tokio::try_join!(fuzz_task, report_task)?; + let result = tokio::try_join!(fuzz_task, report_task)?; + result.0?; + result.1?; Ok(()) } diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index cc9cc584b5..aaf78bbb79 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -184,9 +184,7 @@ impl Config { Config::GenericSupervisor(config) => fuzz::supervisor::spawn(config).await, Config::GenericMerge(config) => merge::generic::spawn(Arc::new(config)).await, Config::GenericReport(config) => { - report::generic::ReportTask::new(&config) - .managed_run() - .await + report::generic::ReportTask::new(config).managed_run().await } } } diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index 7a9fc5d5cf..140d02bfc7 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -149,36 +149,38 @@ impl GeneratorTask { corpus_dir: impl AsRef, output_dir: impl AsRef, ) -> Result<()> { - let mut expand = Expand::new(); - expand - .generated_inputs(&output_dir) - .input_corpus(&corpus_dir) - .generator_exe(&self.config.generator_exe) - .generator_options(&self.config.generator_options); - - if let Some(tools) = &self.config.tools { - expand.tools_dir(&tools.path); - } - utils::reset_tmp_dir(&output_dir).await?; + let mut generator = { + let mut expand = Expand::new(); + expand + .generated_inputs(&output_dir) + .input_corpus(&corpus_dir) + .generator_exe(&self.config.generator_exe) + .generator_options(&self.config.generator_options); + + if let Some(tools) = &self.config.tools { + expand.tools_dir(&tools.path); + } - let generator_path = expand.evaluate_value(&self.config.generator_exe)?; + let generator_path = expand.evaluate_value(&self.config.generator_exe)?; - let mut generator = Command::new(&generator_path); - generator - .kill_on_drop(true) - .env_remove("RUST_LOG") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + let mut generator = Command::new(&generator_path); + generator + .kill_on_drop(true) + .env_remove("RUST_LOG") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); - for arg in expand.evaluate(&self.config.generator_options)? { - generator.arg(arg); - } + for arg in expand.evaluate(&self.config.generator_options)? { + generator.arg(arg); + } - for (k, v) in &self.config.generator_env { - generator.env(k, expand.evaluate_value(v)?); - } + for (k, v) in &self.config.generator_env { + generator.env(k, expand.evaluate_value(v)?); + } + generator + }; info!("Generating test cases with {:?}", generator); let output = generator.spawn()?; diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index b9f3b10297..f7f652af3d 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -150,7 +150,7 @@ impl LibFuzzerFuzzTask { let crash_dir = tempdir()?; let run_id = Uuid::new_v4(); - info!("starting fuzzer run, run_id = {}", run_id); + verbose!("starting fuzzer run, run_id = {}", run_id); let mut inputs = vec![&self.config.inputs.path]; if let Some(readonly_inputs) = &self.config.readonly_inputs { diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index 28b82932cd..a69650b076 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -54,13 +54,13 @@ pub struct Config { pub common: CommonConfig, } -pub struct ReportTask<'a> { - config: &'a Config, +pub struct ReportTask { + config: Config, poller: InputPoller, } -impl<'a> ReportTask<'a> { - pub fn new(config: &'a Config) -> Self { +impl ReportTask { + pub fn new(config: Config) -> Self { let poller = InputPoller::new(); Self { config, poller } } diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index d990a68745..9903c63576 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -5,7 +5,7 @@ use super::crash_report::*; use crate::tasks::{ config::CommonConfig, generic::input_poller::*, heartbeat::*, utils::default_bool_true, }; -use anyhow::Result; +use anyhow::{Context, Result}; use async_trait::async_trait; use futures::stream::StreamExt; use onefuzz::{ @@ -65,6 +65,7 @@ impl ReportTask { Some(x) => x, None => bail!("missing crashes directory"), }; + crashes.init().await?; self.config.unique_reports.init().await?; if let Some(reports) = &self.config.reports { @@ -74,7 +75,13 @@ impl ReportTask { no_repro.init().await?; } - let mut read_dir = tokio::fs::read_dir(&crashes.path).await?; + let mut read_dir = tokio::fs::read_dir(&crashes.path).await.with_context(|| { + format_err!( + "unable to read crashes directory {}", + crashes.path.display() + ) + })?; + while let Some(crash) = read_dir.next().await { processor.process(None, &crash?.path()).await?; } @@ -142,7 +149,9 @@ impl AsanProcessor { Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)), None => None, }; - let input_sha256 = sha256::digest_file(input).await?; + let input_sha256 = sha256::digest_file(input).await.with_context(|| { + format_err!("unable to sha256 digest input file: {}", input.display()) + })?; let test_report = fuzzer .repro( diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index 86ebca06d5..1b9aa003ad 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -68,7 +68,7 @@ impl SyncedDir { anyhow::bail!("File with name '{}' already exists", self.path.display()); } } - Err(_) => fs::create_dir(&self.path).await.with_context(|| { + Err(_) => fs::create_dir_all(&self.path).await.with_context(|| { format!("unable to create local SyncedDir: {}", self.path.display()) }), } From a79912d1b1a419df39660f654b780e3e8a7628cf Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Fri, 15 Jan 2021 09:50:54 -0500 Subject: [PATCH 28/30] set setup_dir to PathBuf::default() if it doesn't exist --- src/agent/onefuzz-agent/src/tasks/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index aaf78bbb79..9849e9a56b 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -34,6 +34,7 @@ pub struct CommonConfig { pub telemetry_key: Option, + #[serde(default)] pub setup_dir: PathBuf, } From bec0680cfe4337c78c57f2f871d878baeeac0b7d Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 19 Jan 2021 13:46:33 -0500 Subject: [PATCH 29/30] ensure we keep the tempdir until after process --- .../src/tasks/generic/input_poller.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index 7ba6b63975..5a01481bb3 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -7,7 +7,7 @@ use anyhow::Result; use futures::stream::StreamExt; use onefuzz::{blob::BlobUrl, jitter::delay_with_jitter, syncdir::SyncedDir}; use reqwest::Url; -use tempfile::tempdir; +use tempfile::{tempdir, TempDir}; use tokio::{fs, time::Duration}; mod callback; @@ -18,12 +18,12 @@ const POLL_INTERVAL: Duration = Duration::from_secs(10); #[cfg(test)] mod tests; -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Debug)] pub enum State { Ready, Polled(Option), Parsed(M, Url), - Downloaded(M, Url, PathBuf), + Downloaded(M, Url, PathBuf, TempDir), Processed(M), } @@ -164,7 +164,7 @@ impl InputPoller { verbose!("Input queue empty, sleeping"); delay_with_jitter(POLL_INTERVAL).await; } - State::Downloaded(_msg, _url, input) => { + State::Downloaded(_msg, _url, input, _tempdir) => { info!("Processing downloaded input: {:?}", input); } _ => {} @@ -248,10 +248,13 @@ impl InputPoller { .download(url.clone(), download_dir.path()) .await?; - self.set_state(Downloaded(msg, url, input)); + self.set_state(Downloaded(msg, url, input, download_dir)); } } - (Downloaded(msg, url, input), Process(processor)) => { + // NOTE: _download_dir is a TempDir, which the physical path gets + // deleted automatically upon going out of scope. Keep it in-scope until + // here. + (Downloaded(msg, url, input, _download_dir), Process(processor)) => { processor.process(Some(url), &input).await?; self.set_state(Processed(msg)); From a62d8d6b707281f3ab207996a9e32ec7818393b5 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 19 Jan 2021 16:58:36 -0500 Subject: [PATCH 30/30] use az-cli user agent when using the AZ_CLI wrappers --- src/deployment/registration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/deployment/registration.py b/src/deployment/registration.py index 48b6636cc5..59cefa4932 100644 --- a/src/deployment/registration.py +++ b/src/deployment/registration.py @@ -13,6 +13,7 @@ from uuid import UUID, uuid4 import requests +from azure.cli.core.utils import get_az_rest_user_agent from azure.common.client_factory import get_client_from_cli_profile from azure.common.credentials import get_cli_profile from azure.graphrbac import GraphRbacManagementClient @@ -49,6 +50,7 @@ def query_microsoft_graph( headers = { "Authorization": "%s %s" % (token_type, access_token), "Content-Type": "application/json", + "User-Agent": get_az_rest_user_agent(), } response = requests.request( method=method, url=url, headers=headers, params=params, json=body