diff --git a/src/agent/onefuzz-agent/src/debug/cmd.rs b/src/agent/onefuzz-agent/src/debug/cmd.rs index 9d14c5327b..15cf6d0ab4 100644 --- a/src/agent/onefuzz-agent/src/debug/cmd.rs +++ b/src/agent/onefuzz-agent/src/debug/cmd.rs @@ -4,25 +4,21 @@ use anyhow::Result; use clap::{App, SubCommand}; -pub fn run(args: &clap::ArgMatches) -> Result<()> { +use crate::{debug::libfuzzer_merge, local::common::add_common_config}; + +const LIBFUZZER_MERGE: &str = "libfuzzer-merge"; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { match args.subcommand() { - ("generic-crash-report", Some(sub)) => crate::debug::generic_crash_report::run(sub)?, - ("libfuzzer-coverage", Some(sub)) => crate::debug::libfuzzer_coverage::run(sub)?, - ("libfuzzer-crash-report", Some(sub)) => crate::debug::libfuzzer_crash_report::run(sub)?, - ("libfuzzer-fuzz", Some(sub)) => crate::debug::libfuzzer_fuzz::run(sub)?, - ("libfuzzer-merge", Some(sub)) => crate::debug::libfuzzer_merge::run(sub)?, - _ => println!("missing subcommand\nUSAGE : {}", args.usage()), + (LIBFUZZER_MERGE, Some(sub)) => libfuzzer_merge::run(sub).await, + _ => { + anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); + } } - - Ok(()) } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("debug") +pub fn args(name: &str) -> App<'static, 'static> { + SubCommand::with_name(name) .about("unsupported internal debugging commands") - .subcommand(crate::debug::generic_crash_report::args()) - .subcommand(crate::debug::libfuzzer_coverage::args()) - .subcommand(crate::debug::libfuzzer_crash_report::args()) - .subcommand(crate::debug::libfuzzer_fuzz::args()) - .subcommand(crate::debug::libfuzzer_merge::args()) + .subcommand(add_common_config(libfuzzer_merge::args(LIBFUZZER_MERGE))) } diff --git a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs b/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs deleted file mode 100644 index d10543ade4..0000000000 --- a/src/agent/onefuzz-agent/src/debug/generic_crash_report.rs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use crate::tasks::{ - config::CommonConfig, - report::generic::{Config, GenericReportProcessor}, - utils::parse_key_value, -}; -use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; -use tokio::runtime::Runtime; -use url::Url; -use uuid::Uuid; - -async fn run_impl(input: String, config: Config) -> Result<()> { - let input_path = Path::new(&input); - let test_url = Url::parse("https://contoso.com/sample-container/blob.txt")?; - let heartbeat_client = config.common.init_heartbeat().await?; - let processor = GenericReportProcessor::new(&config, heartbeat_client); - let result = processor.test_input(test_url, input_path).await?; - println!("{:#?}", result); - Ok(()) -} - -pub fn run(args: &clap::ArgMatches) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let setup_dir = value_t!(args, "setup_dir", PathBuf)?; - let input = value_t!(args, "input", String)?; - let target_timeout = value_t!(args, "target_timeout", u64).ok(); - let check_retry_count = value_t!(args, "check_retry_count", u64)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - let check_asan_log = args.is_present("check_asan_log"); - let check_debugger = !args.is_present("disable_check_debugger"); - - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - - let config = Config { - target_exe, - target_env, - target_options, - target_timeout, - check_asan_log, - check_debugger, - check_retry_count, - crashes: None, - input_queue: None, - no_repro: None, - reports: None, - unique_reports: SyncedDir { - path: "unique_reports".into(), - url: BlobContainerUrl::new(url::Url::parse("https://contoso.com/unique_reports")?)?, - }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - setup_dir, - }, - }; - - let mut rt = Runtime::new()?; - rt.block_on(async { run_impl(input, config).await })?; - - Ok(()) -} - -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("generic-crash-report") - .about("execute a local-only generic crash report") - .arg( - Arg::with_name("setup_dir") - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name("input").takes_value(true).required(true)) - .arg( - Arg::with_name("disable_check_debugger") - .takes_value(false) - .long("disable_check_debugger"), - ) - .arg( - Arg::with_name("check_asan_log") - .takes_value(false) - .long("check_asan_log"), - ) - .arg( - Arg::with_name("check_retry_count") - .takes_value(true) - .long("check_retry_count") - .default_value("0"), - ) - .arg( - Arg::with_name("target_timeout") - .takes_value(true) - .long("target_timeout") - .default_value("5"), - ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) - .arg( - Arg::with_name("target_options") - .long("target_options") - .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .default_value("{input}") - .help("Supports hyphens. Recommendation: Set target_env first"), - ) -} diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs deleted file mode 100644 index 447d19d5bf..0000000000 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use crate::tasks::{ - config::CommonConfig, - coverage::libfuzzer_coverage::{Config, CoverageProcessor}, - utils::parse_key_value, -}; -use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; -use tokio::runtime::Runtime; -use url::Url; -use uuid::Uuid; - -async fn run_impl(input: String, config: Config) -> Result<()> { - let mut processor = CoverageProcessor::new(Arc::new(config)) - .await - .map_err(|e| format_err!("coverage processor failed: {:?}", e))?; - let input_path = Path::new(&input); - processor - .test_input(input_path) - .await - .map_err(|e| format_err!("test input failed {:?}", e))?; - let info = processor - .total - .info() - .await - .map_err(|e| format_err!("coverage_info failed {:?}", e))?; - println!("{:?}", info); - Ok(()) -} - -pub fn run(args: &clap::ArgMatches) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let setup_dir = value_t!(args, "setup_dir", PathBuf)?; - let input = value_t!(args, "input", String)?; - let result_dir = value_t!(args, "result_dir", String)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - - // this happens during setup, not during runtime - let check_fuzzer_help = true; - - let config = Config { - target_exe, - target_env, - target_options, - check_fuzzer_help, - input_queue: None, - readonly_inputs: vec![], - coverage: SyncedDir { - path: result_dir.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/coverage")?)?, - }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - setup_dir, - }, - }; - - let mut rt = Runtime::new()?; - rt.block_on(run_impl(input, config))?; - - Ok(()) -} - -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-coverage") - .about("execute a local-only libfuzzer coverage task") - .arg( - Arg::with_name("setup_dir") - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name("input").takes_value(true).required(true)) - .arg( - Arg::with_name("result_dir") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) - .arg( - Arg::with_name("target_options") - .long("target_options") - .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .default_value("{input}") - .help("Supports hyphens. Recommendation: Set target_env first"), - ) -} diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs deleted file mode 100644 index 204b444db6..0000000000 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use crate::tasks::{ - config::CommonConfig, - report::libfuzzer_report::{AsanProcessor, Config}, - utils::parse_key_value, -}; -use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; -use tokio::runtime::Runtime; -use url::Url; -use uuid::Uuid; - -async fn run_impl(input: String, config: Config) -> Result<()> { - let task = AsanProcessor::new(Arc::new(config)).await?; - - let test_url = Url::parse("https://contoso.com/sample-container/blob.txt")?; - let input_path = Path::new(&input); - let result = task.test_input(test_url, &input_path).await; - println!("{:#?}", result); - Ok(()) -} - -pub fn run(args: &clap::ArgMatches) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let setup_dir = value_t!(args, "setup_dir", PathBuf)?; - let input = value_t!(args, "input", String)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - let target_timeout = value_t!(args, "target_timeout", u64).ok(); - let check_retry_count = value_t!(args, "check_retry_count", u64)?; - - // this happens during setup, not during runtime - let check_fuzzer_help = true; - - let config = Config { - target_exe, - target_env, - target_options, - target_timeout, - check_retry_count, - check_fuzzer_help, - input_queue: None, - crashes: None, - reports: None, - no_repro: None, - unique_reports: SyncedDir { - path: "unique_reports".into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/unique_reports")?)?, - }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - setup_dir, - }, - }; - - let mut rt = Runtime::new()?; - rt.block_on(async { run_impl(input, config).await })?; - - Ok(()) -} - -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-crash-report") - .about("execute a local-only libfuzzer crash report task") - .arg( - Arg::with_name("setup_dir") - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name("input").takes_value(true).required(true)) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) - .arg( - Arg::with_name("target_options") - .long("target_options") - .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set target_env first"), - ) - .arg( - Arg::with_name("target_timeout") - .takes_value(true) - .long("target_timeout"), - ) - .arg( - Arg::with_name("check_retry_count") - .takes_value(true) - .long("check_retry_count") - .default_value("0"), - ) -} diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs deleted file mode 100644 index fdc345ed9d..0000000000 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use crate::tasks::{ - config::CommonConfig, - fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, - utils::parse_key_value, -}; -use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{collections::HashMap, path::PathBuf}; -use tokio::runtime::Runtime; -use url::Url; -use uuid::Uuid; - -async fn run_impl(config: Config) -> Result<()> { - let fuzzer = LibFuzzerFuzzTask::new(config)?; - let result = fuzzer.start_fuzzer_monitor(0, None).await?; - println!("{:#?}", result); - Ok(()) -} - -pub fn run(args: &clap::ArgMatches) -> Result<()> { - let crashes_dir = value_t!(args, "crashes_dir", String)?; - let inputs_dir = value_t!(args, "inputs_dir", String)?; - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let setup_dir = value_t!(args, "setup_dir", PathBuf)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - - // this happens during setup, not during runtime - let check_fuzzer_help = true; - - let expect_crash_on_failure = args.is_present("expect_crash_on_failure"); - - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } - - let readonly_inputs = None; - let target_workers = Some(1); - - let inputs = SyncedDir { - path: inputs_dir.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/inputs")?)?, - }; - - let crashes = SyncedDir { - path: crashes_dir.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/crashes")?)?, - }; - - let ensemble_sync_delay = None; - - let config = Config { - inputs, - readonly_inputs, - crashes, - target_exe, - target_env, - target_options, - target_workers, - ensemble_sync_delay, - check_fuzzer_help, - expect_crash_on_failure, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - setup_dir, - }, - }; - - let mut rt = Runtime::new()?; - rt.block_on(async { run_impl(config).await })?; - - Ok(()) -} - -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-fuzz") - .about("execute a local-only libfuzzer crash report task") - .arg( - Arg::with_name("setup_dir") - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) - .arg( - Arg::with_name("target_options") - .long("target_options") - .takes_value(true) - .multiple(true) - .allow_hyphen_values(true) - .help("Supports hyphens. Recommendation: Set target_env first"), - ) - .arg( - Arg::with_name("inputs_dir") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name("crashes_dir") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name("expect_crash_on_failure") - .takes_value(false) - .long("expect_crash_on_failure"), - ) -} diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs index 274cb8f8a6..7d396fd446 100644 --- a/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs @@ -1,35 +1,27 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::tasks::{ - config::CommonConfig, - merge::libfuzzer_merge::{merge_inputs, Config}, - utils::parse_key_value, +use crate::{ + local::common::{ + add_cmd_options, build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, + }, + tasks::merge::libfuzzer_merge::{merge_inputs, Config}, }; use anyhow::Result; use clap::{App, Arg, SubCommand}; -use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir}; -use std::{collections::HashMap, path::PathBuf, sync::Arc}; -use tokio::runtime::Runtime; -use url::Url; -use uuid::Uuid; +use onefuzz::syncdir::SyncedDir; +use std::sync::Arc; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); -pub fn run(args: &clap::ArgMatches) -> Result<()> { - let target_exe = value_t!(args, "target_exe", PathBuf)?; - let setup_dir = value_t!(args, "setup_dir", PathBuf)?; let inputs = value_t!(args, "inputs", String)?; let unique_inputs = value_t!(args, "unique_inputs", String)?; - let target_options = args.values_of_lossy("target_options").unwrap_or_default(); - - // this happens during setup, not during runtime - let check_fuzzer_help = true; - - let mut target_env = HashMap::new(); - for opt in args.values_of_lossy("target_env").unwrap_or_default() { - let (k, v) = parse_key_value(opt)?; - target_env.insert(k, v); - } + let check_fuzzer_help = false; + let common = build_common_config(args)?; let config = Arc::new(Config { target_exe, target_env, @@ -38,56 +30,29 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { input_queue: None, inputs: vec![SyncedDir { path: inputs.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/inputs")?)?, + url: None, }], unique_inputs: SyncedDir { path: unique_inputs.into(), - url: BlobContainerUrl::new(Url::parse("https://contoso.com/unique_inputs")?)?, - }, - common: CommonConfig { - heartbeat_queue: None, - instrumentation_key: None, - telemetry_key: None, - job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), - task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - instance_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - setup_dir, + url: None, }, + common, preserve_existing_outputs: true, }); - let mut rt = Runtime::new()?; - rt.block_on(merge_inputs( - config.clone(), - vec![config.inputs[0].path.clone()], - ))?; - + let results = merge_inputs(config.clone(), vec![config.clone().inputs[0].path.clone()]).await?; + println!("{:#?}", results); Ok(()) } -pub fn args() -> App<'static, 'static> { - SubCommand::with_name("libfuzzer-merge") - .about("execute a local-only libfuzzer merge task") - .arg( - Arg::with_name("setup_dir") - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name("target_exe") - .takes_value(true) - .required(true), - ) - .arg(Arg::with_name("inputs").takes_value(true).required(true)) +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("execute a local-only libfuzzer merge task"); + + app = add_cmd_options(CmdType::Target, true, true, true, app); + app.arg(Arg::with_name("inputs").takes_value(true).required(true)) .arg( Arg::with_name("unique_inputs") .takes_value(true) .required(true), ) - .arg( - Arg::with_name("target_env") - .long("target_env") - .takes_value(true) - .multiple(true), - ) } diff --git a/src/agent/onefuzz-agent/src/debug/mod.rs b/src/agent/onefuzz-agent/src/debug/mod.rs index d84cc95080..0a7fefd34b 100644 --- a/src/agent/onefuzz-agent/src/debug/mod.rs +++ b/src/agent/onefuzz-agent/src/debug/mod.rs @@ -2,8 +2,4 @@ // Licensed under the MIT License. pub mod cmd; -pub mod generic_crash_report; -pub mod libfuzzer_coverage; -pub mod libfuzzer_crash_report; -pub mod libfuzzer_fuzz; pub mod libfuzzer_merge; diff --git a/src/agent/onefuzz-agent/src/local/cmd.rs b/src/agent/onefuzz-agent/src/local/cmd.rs new file mode 100644 index 0000000000..4bf17e4005 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/cmd.rs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use anyhow::Result; +use clap::{App, SubCommand}; + +use crate::local::{ + common::add_common_config, generic_crash_report, generic_generator, libfuzzer, + libfuzzer_coverage, libfuzzer_crash_report, libfuzzer_fuzz, radamsa, +}; + +const RADAMSA: &str = "radamsa"; +const LIBFUZZER: &str = "libfuzzer"; +const LIBFUZZER_FUZZ: &str = "libfuzzer-fuzz"; +const LIBFUZZER_CRASH_REPORT: &str = "libfuzzer-crash-report"; +const LIBFUZZER_COVERAGE: &str = "libfuzzer-coverage"; +const GENERIC_CRASH_REPORT: &str = "generic-crash-report"; +const GENERIC_GENERATOR: &str = "generic-generator"; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + match args.subcommand() { + (RADAMSA, Some(sub)) => radamsa::run(sub).await, + (LIBFUZZER, Some(sub)) => libfuzzer::run(sub).await, + (LIBFUZZER_FUZZ, Some(sub)) => libfuzzer_fuzz::run(sub).await, + (LIBFUZZER_COVERAGE, Some(sub)) => libfuzzer_coverage::run(sub).await, + (LIBFUZZER_CRASH_REPORT, Some(sub)) => libfuzzer_crash_report::run(sub).await, + (GENERIC_CRASH_REPORT, Some(sub)) => generic_crash_report::run(sub).await, + (GENERIC_GENERATOR, Some(sub)) => generic_generator::run(sub).await, + _ => { + anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); + } + } +} + +pub fn args(name: &str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("pre-release local fuzzing") + .subcommand(add_common_config(radamsa::args(RADAMSA))) + .subcommand(add_common_config(libfuzzer::args(LIBFUZZER))) + .subcommand(add_common_config(libfuzzer_fuzz::args(LIBFUZZER_FUZZ))) + .subcommand(add_common_config(libfuzzer_coverage::args( + LIBFUZZER_COVERAGE, + ))) + .subcommand(add_common_config(libfuzzer_crash_report::args( + LIBFUZZER_CRASH_REPORT, + ))) + .subcommand(add_common_config(generic_crash_report::args( + GENERIC_CRASH_REPORT, + ))) + .subcommand(add_common_config(generic_generator::args( + GENERIC_GENERATOR, + ))) +} diff --git a/src/agent/onefuzz-agent/src/local/common.rs b/src/agent/onefuzz-agent/src/local/common.rs new file mode 100644 index 0000000000..2fcc292db1 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/common.rs @@ -0,0 +1,180 @@ +use crate::tasks::config::CommonConfig; +use crate::tasks::utils::parse_key_value; +use anyhow::Result; +use clap::{App, Arg, ArgMatches}; +use std::{collections::HashMap, path::PathBuf}; + +use uuid::Uuid; + +pub const SETUP_DIR: &str = "setup_dir"; +pub const INPUTS_DIR: &str = "inputs_dir"; +pub const CRASHES_DIR: &str = "crashes_dir"; +pub const TARGET_WORKERS: &str = "target_workers"; +pub const REPORTS_DIR: &str = "reports_dir"; +pub const NO_REPRO_DIR: &str = "no_repro_dir"; +pub const TARGET_TIMEOUT: &str = "target_timeout"; +pub const CHECK_RETRY_COUNT: &str = "check_retry_count"; +pub const DISABLE_CHECK_QUEUE: &str = "disable_check_queue"; +pub const UNIQUE_REPORTS_DIR: &str = "unique_reports_dir"; +pub const COVERAGE_DIR: &str = "coverage_dir"; +pub const READONLY_INPUTS: &str = "readonly_inputs_dir"; +pub const CHECK_ASAN_LOG: &str = "check_asan_log"; +pub const TOOLS_DIR: &str = "tools_dir"; +pub const RENAME_OUTPUT: &str = "rename_output"; +pub const CHECK_FUZZER_HELP: &str = "check_fuzzer_help"; + +pub const TARGET_EXE: &str = "target_exe"; +pub const TARGET_ENV: &str = "target_env"; +pub const TARGET_OPTIONS: &str = "target_options"; +pub const SUPERVISOR_EXE: &str = "supervisor_exe"; +pub const SUPERVISOR_ENV: &str = "supervisor_env"; +pub const SUPERVISOR_OPTIONS: &str = "supervisor_options"; +pub const GENERATOR_EXE: &str = "generator_exe"; +pub const GENERATOR_ENV: &str = "generator_env"; +pub const GENERATOR_OPTIONS: &str = "generator_options"; + +pub enum CmdType { + Target, + Generator, + Supervisor, +} + +pub fn add_cmd_options( + cmd_type: CmdType, + exe: bool, + arg: bool, + env: bool, + mut app: App<'static, 'static>, +) -> App<'static, 'static> { + let (exe_name, env_name, arg_name) = match cmd_type { + CmdType::Target => (TARGET_EXE, TARGET_ENV, TARGET_OPTIONS), + CmdType::Supervisor => (SUPERVISOR_EXE, SUPERVISOR_ENV, SUPERVISOR_OPTIONS), + CmdType::Generator => (GENERATOR_EXE, GENERATOR_ENV, GENERATOR_OPTIONS), + }; + + if exe { + app = app.arg(Arg::with_name(exe_name).takes_value(true).required(true)); + } + if env { + app = app.arg( + Arg::with_name(env_name) + .long(env_name) + .takes_value(true) + .multiple(true), + ) + } + if arg { + app = app.arg( + Arg::with_name(arg_name) + .long(arg_name) + .takes_value(true) + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), + ) + } + app +} + +pub fn get_cmd_exe(cmd_type: CmdType, args: &clap::ArgMatches<'_>) -> Result { + let name = match cmd_type { + CmdType::Target => TARGET_EXE, + CmdType::Supervisor => SUPERVISOR_EXE, + CmdType::Generator => GENERATOR_EXE, + }; + + let exe = value_t!(args, name, String)?; + Ok(exe) +} + +pub fn get_cmd_arg(cmd_type: CmdType, args: &clap::ArgMatches<'_>) -> Vec { + let name = match cmd_type { + CmdType::Target => TARGET_OPTIONS, + CmdType::Supervisor => SUPERVISOR_OPTIONS, + CmdType::Generator => GENERATOR_OPTIONS, + }; + + args.values_of_lossy(name).unwrap_or_default() +} + +pub fn get_cmd_env( + cmd_type: CmdType, + args: &clap::ArgMatches<'_>, +) -> Result> { + let env_name = match cmd_type { + CmdType::Target => TARGET_ENV, + CmdType::Supervisor => SUPERVISOR_ENV, + CmdType::Generator => GENERATOR_ENV, + }; + + let mut env = HashMap::new(); + for opt in args.values_of_lossy(env_name).unwrap_or_default() { + let (k, v) = parse_key_value(opt)?; + env.insert(k, v); + } + Ok(env) +} + +pub fn add_common_config(app: App<'static, 'static>) -> App<'static, 'static> { + app.arg( + Arg::with_name("job_id") + .long("job_id") + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name("task_id") + .long("task_id") + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name("instance_id") + .long("instance_id") + .takes_value(true) + .required(false), + ) + .arg( + Arg::with_name("setup_dir") + .long("setup_dir") + .takes_value(true) + .required(false), + ) +} + +fn get_uuid(name: &str, args: &ArgMatches<'_>) -> Result { + match value_t!(args, name, String) { + Ok(x) => Uuid::parse_str(&x) + .map_err(|x| format_err!("invalid {}. uuid expected. {})", name, x)), + Err(_) => Ok(Uuid::nil()), + } +} + +pub fn build_common_config(args: &ArgMatches<'_>) -> Result { + let job_id = get_uuid("job_id", args)?; + let task_id = get_uuid("task_id", args)?; + let instance_id = get_uuid("instance_id", args)?; + + let setup_dir = if args.is_present(SETUP_DIR) { + value_t!(args, SETUP_DIR, PathBuf)? + } else { + if args.is_present(TARGET_EXE) { + value_t!(args, TARGET_EXE, PathBuf)? + .parent() + .map(|x| x.to_path_buf()) + .unwrap_or_default() + } else { + PathBuf::default() + } + }; + + let config = CommonConfig { + heartbeat_queue: None, + instrumentation_key: None, + telemetry_key: None, + job_id, + task_id, + instance_id, + setup_dir, + }; + Ok(config) +} diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs new file mode 100644 index 0000000000..4ef3d976c0 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_ASAN_LOG, + CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, + TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + }, + tasks::report::generic::{Config, ReportTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + + let crashes = Some(value_t!(args, CRASHES_DIR, PathBuf)?.into()); + let reports = if args.is_present(REPORTS_DIR) { + Some(value_t!(args, REPORTS_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + let no_repro = if args.is_present(NO_REPRO_DIR) { + Some(value_t!(args, NO_REPRO_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); + + let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); + + let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; + let check_queue = !args.is_present(DISABLE_CHECK_QUEUE); + let check_asan_log = args.is_present(CHECK_ASAN_LOG); + let check_debugger = !args.is_present("disable_check_debugger"); + + let common = build_common_config(args)?; + + let config = Config { + target_exe, + target_env, + target_options, + target_timeout, + check_asan_log, + check_debugger, + check_retry_count, + check_queue, + crashes, + input_queue: None, + no_repro, + reports, + unique_reports, + common, + }; + + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_report_config(args)?; + ReportTask::new(config).local_run().await +} + +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .default_value("{input}") + .long(TARGET_OPTIONS) + .takes_value(true) + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), + Arg::with_name(CRASHES_DIR) + .long(CRASHES_DIR) + .takes_value(true) + .required(true), + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) + .takes_value(true) + .required(false), + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) + .takes_value(true) + .required(false), + Arg::with_name(UNIQUE_REPORTS_DIR) + .long(UNIQUE_REPORTS_DIR) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT) + .default_value("30"), + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + Arg::with_name(DISABLE_CHECK_QUEUE) + .takes_value(false) + .long(DISABLE_CHECK_QUEUE), + Arg::with_name(CHECK_ASAN_LOG) + .takes_value(false) + .long(CHECK_ASAN_LOG), + Arg::with_name("disable_check_debugger") + .takes_value(false) + .long("disable_check_debugger"), + ] +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("execute a local-only generic crash report") + .args(&build_shared_args()) +} diff --git a/src/agent/onefuzz-agent/src/local/generic_generator.rs b/src/agent/onefuzz-agent/src/local/generic_generator.rs new file mode 100644 index 0000000000..e452078660 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/generic_generator.rs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_ASAN_LOG, + CHECK_RETRY_COUNT, CRASHES_DIR, GENERATOR_ENV, GENERATOR_EXE, GENERATOR_OPTIONS, + READONLY_INPUTS, RENAME_OUTPUT, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, + TOOLS_DIR, + }, + tasks::fuzz::generator::{Config, GeneratorTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { + let crashes = value_t!(args, CRASHES_DIR, PathBuf)?.into(); + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_options = get_cmd_arg(CmdType::Target, args); + let target_env = get_cmd_env(CmdType::Target, args)?; + + let generator_exe = get_cmd_exe(CmdType::Generator, args)?; + let generator_options = get_cmd_arg(CmdType::Generator, args); + let generator_env = get_cmd_env(CmdType::Generator, args)?; + + let readonly_inputs = values_t!(args, READONLY_INPUTS, PathBuf)? + .iter() + .map(|x| x.to_owned().into()) + .collect(); + + let rename_output = args.is_present(RENAME_OUTPUT); + let check_asan_log = args.is_present(CHECK_ASAN_LOG); + let check_debugger = !args.is_present("disable_check_debugger"); + let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; + let target_timeout = Some(value_t!(args, TARGET_TIMEOUT, u64)?); + + let tools = if args.is_present(TOOLS_DIR) { + Some(value_t!(args, TOOLS_DIR, PathBuf)?.into()) + } else { + None + }; + + let ensemble_sync_delay = None; + let common = build_common_config(args)?; + let config = Config { + tools, + generator_exe, + generator_env, + generator_options, + target_exe, + target_env, + target_options, + target_timeout, + readonly_inputs, + crashes, + ensemble_sync_delay, + check_asan_log, + check_debugger, + check_retry_count, + rename_output, + common, + }; + + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_fuzz_config(args)?; + GeneratorTask::new(config).run().await +} + +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .default_value("{input}") + .long(TARGET_OPTIONS) + .takes_value(true) + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), + Arg::with_name(GENERATOR_EXE) + .long(GENERATOR_EXE) + .default_value("radamsa") + .takes_value(true) + .required(true), + Arg::with_name(GENERATOR_ENV) + .long(GENERATOR_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(GENERATOR_OPTIONS) + .long(GENERATOR_OPTIONS) + .takes_value(true) + .value_delimiter(" ") + .default_value("-H sha256 -o {generated_inputs}/input-%h.%s -n 100 -r {input_corpus}") + .help("Use a quoted string with space separation to denote multiple arguments"), + Arg::with_name(CRASHES_DIR) + .takes_value(true) + .required(true) + .long(CRASHES_DIR), + Arg::with_name(READONLY_INPUTS) + .takes_value(true) + .required(true) + .multiple(true) + .long(READONLY_INPUTS), + Arg::with_name(TOOLS_DIR).takes_value(true).long(TOOLS_DIR), + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + Arg::with_name(CHECK_ASAN_LOG) + .takes_value(false) + .long(CHECK_ASAN_LOG), + Arg::with_name(RENAME_OUTPUT) + .takes_value(false) + .long(RENAME_OUTPUT), + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT) + .default_value("30"), + Arg::with_name("disable_check_debugger") + .takes_value(false) + .long("disable_check_debugger"), + ] +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("execute a local-only generator fuzzing task") + .args(&build_shared_args()) +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer.rs b/src/agent/onefuzz-agent/src/local/libfuzzer.rs new file mode 100644 index 0000000000..9bfb273ae9 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/libfuzzer.rs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::{ + common::COVERAGE_DIR, + libfuzzer_coverage::{build_coverage_config, build_shared_args as build_coverage_args}, + libfuzzer_crash_report::{build_report_config, build_shared_args as build_crash_args}, + libfuzzer_fuzz::{build_fuzz_config, build_shared_args as build_fuzz_args}, + }, + tasks::{ + coverage::libfuzzer_coverage::CoverageTask, fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask, + report::libfuzzer_report::ReportTask, + }, +}; +use anyhow::Result; +use clap::{App, SubCommand}; +use std::collections::HashSet; +use tokio::task::spawn; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let fuzz_config = build_fuzz_config(args)?; + let fuzzer = LibFuzzerFuzzTask::new(fuzz_config)?; + let fuzz_task = spawn(async move { fuzzer.run().await }); + + let report_config = build_report_config(args)?; + let report = ReportTask::new(report_config); + let report_task = spawn(async move { report.local_run().await }); + + if args.is_present(COVERAGE_DIR) { + let coverage_config = build_coverage_config(args, true)?; + let coverage = CoverageTask::new(coverage_config); + let coverage_task = spawn(async move { coverage.local_run().await }); + + let result = tokio::try_join!(fuzz_task, report_task, coverage_task)?; + result.0?; + result.1?; + result.2?; + } else { + let result = tokio::try_join!(fuzz_task, report_task)?; + result.0?; + result.1?; + } + + Ok(()) +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("run a local libfuzzer & crash reporting task"); + + let mut used = HashSet::new(); + for args in &[ + build_fuzz_args(), + build_crash_args(), + build_coverage_args(true), + ] { + for arg in args { + if used.contains(arg.b.name) { + continue; + } + used.insert(arg.b.name.to_string()); + app = app.arg(arg); + } + } + + app +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs new file mode 100644 index 0000000000..09ed828aea --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_coverage.rs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_FUZZER_HELP, + COVERAGE_DIR, INPUTS_DIR, READONLY_INPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, + }, + tasks::coverage::libfuzzer_coverage::{Config, CoverageTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +pub fn build_coverage_config(args: &clap::ArgMatches<'_>, local_job: bool) -> Result { + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + + let readonly_inputs = if local_job { + vec![value_t!(args, INPUTS_DIR, PathBuf)?.into()] + } else { + values_t!(args, READONLY_INPUTS, PathBuf)? + .iter() + .map(|x| x.to_owned().into()) + .collect() + }; + + let coverage = value_t!(args, COVERAGE_DIR, PathBuf)?.into(); + let check_fuzzer_help = args.is_present(CHECK_FUZZER_HELP); + + let common = build_common_config(args)?; + let config = Config { + target_exe, + target_env, + target_options, + check_fuzzer_help, + input_queue: None, + readonly_inputs, + coverage, + common, + check_queue: false, + }; + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_coverage_config(args, false)?; + + let task = CoverageTask::new(config); + task.local_run().await +} + +pub fn build_shared_args(local_job: bool) -> Vec> { + let mut args = vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), + Arg::with_name(COVERAGE_DIR) + .takes_value(true) + .required(!local_job) + .long(COVERAGE_DIR), + Arg::with_name(CHECK_FUZZER_HELP) + .takes_value(false) + .long(CHECK_FUZZER_HELP), + ]; + if local_job { + args.push( + Arg::with_name(INPUTS_DIR) + .long(INPUTS_DIR) + .takes_value(true) + .required(true), + ) + } else { + args.push( + Arg::with_name(READONLY_INPUTS) + .takes_value(true) + .required(true) + .long(READONLY_INPUTS) + .multiple(true), + ) + } + args +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("execute a local-only libfuzzer coverage task") + .args(&build_shared_args(false)) +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs new file mode 100644 index 0000000000..7a8a9ee8b5 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_FUZZER_HELP, + CHECK_RETRY_COUNT, CRASHES_DIR, DISABLE_CHECK_QUEUE, NO_REPRO_DIR, REPORTS_DIR, TARGET_ENV, + TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, UNIQUE_REPORTS_DIR, + }, + tasks::report::libfuzzer_report::{Config, ReportTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + + let crashes = Some(value_t!(args, CRASHES_DIR, PathBuf)?.into()); + let reports = if args.is_present(REPORTS_DIR) { + Some(value_t!(args, REPORTS_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + let no_repro = if args.is_present(NO_REPRO_DIR) { + Some(value_t!(args, NO_REPRO_DIR, PathBuf)?).map(|x| x.into()) + } else { + None + }; + let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); + + let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); + + let check_retry_count = value_t!(args, CHECK_RETRY_COUNT, u64)?; + let check_queue = !args.is_present(DISABLE_CHECK_QUEUE); + let check_fuzzer_help = args.is_present(CHECK_FUZZER_HELP); + + let common = build_common_config(args)?; + let config = Config { + target_exe, + target_env, + target_options, + target_timeout, + check_retry_count, + check_fuzzer_help, + input_queue: None, + check_queue, + crashes, + reports, + no_repro, + unique_reports, + common, + }; + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_report_config(args)?; + ReportTask::new(config).local_run().await +} + +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), + Arg::with_name(CRASHES_DIR) + .long(CRASHES_DIR) + .takes_value(true) + .required(true), + Arg::with_name(REPORTS_DIR) + .long(REPORTS_DIR) + .takes_value(true) + .required(false), + Arg::with_name(NO_REPRO_DIR) + .long(NO_REPRO_DIR) + .takes_value(true) + .required(false), + Arg::with_name(UNIQUE_REPORTS_DIR) + .long(UNIQUE_REPORTS_DIR) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_TIMEOUT) + .takes_value(true) + .long(TARGET_TIMEOUT), + Arg::with_name(CHECK_RETRY_COUNT) + .takes_value(true) + .long(CHECK_RETRY_COUNT) + .default_value("0"), + Arg::with_name(DISABLE_CHECK_QUEUE) + .takes_value(false) + .long(DISABLE_CHECK_QUEUE), + Arg::with_name(CHECK_FUZZER_HELP) + .takes_value(false) + .long(CHECK_FUZZER_HELP), + ] +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("execute a local-only libfuzzer crash report task") + .args(&build_shared_args()) +} diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs new file mode 100644 index 0000000000..0e859d23b9 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_fuzz.rs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::common::{ + build_common_config, get_cmd_arg, get_cmd_env, get_cmd_exe, CmdType, CHECK_FUZZER_HELP, + CRASHES_DIR, INPUTS_DIR, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, + }, + tasks::fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use std::path::PathBuf; + +const DISABLE_EXPECT_CRASH_ON_FAILURE: &str = "disable_expect_crash_on_failure"; + +pub fn build_fuzz_config(args: &clap::ArgMatches<'_>) -> Result { + let crashes = value_t!(args, CRASHES_DIR, PathBuf)?.into(); + let inputs = value_t!(args, INPUTS_DIR, PathBuf)?.into(); + + let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); + let target_env = get_cmd_env(CmdType::Target, args)?; + let target_options = get_cmd_arg(CmdType::Target, args); + + let target_workers = value_t!(args, "target_workers", u64).unwrap_or_default(); + let readonly_inputs = None; + let check_fuzzer_help = args.is_present(CHECK_FUZZER_HELP); + let expect_crash_on_failure = !args.is_present(DISABLE_EXPECT_CRASH_ON_FAILURE); + + let ensemble_sync_delay = None; + let common = build_common_config(args)?; + let config = Config { + inputs, + readonly_inputs, + crashes, + target_exe, + target_env, + target_options, + target_workers, + ensemble_sync_delay, + expect_crash_on_failure, + check_fuzzer_help, + common, + }; + + Ok(config) +} + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config = build_fuzz_config(args)?; + LibFuzzerFuzzTask::new(config)?.run().await +} + +pub fn build_shared_args() -> Vec> { + vec![ + Arg::with_name(TARGET_EXE) + .long(TARGET_EXE) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_ENV) + .long(TARGET_ENV) + .takes_value(true) + .multiple(true), + Arg::with_name(TARGET_OPTIONS) + .long(TARGET_OPTIONS) + .takes_value(true) + .value_delimiter(" ") + .help("Use a quoted string with space separation to denote multiple arguments"), + Arg::with_name(INPUTS_DIR) + .long(INPUTS_DIR) + .takes_value(true) + .required(true), + Arg::with_name(CRASHES_DIR) + .long(CRASHES_DIR) + .takes_value(true) + .required(true), + Arg::with_name(TARGET_WORKERS) + .long(TARGET_WORKERS) + .takes_value(true), + Arg::with_name(CHECK_FUZZER_HELP) + .takes_value(false) + .long(CHECK_FUZZER_HELP), + Arg::with_name(DISABLE_EXPECT_CRASH_ON_FAILURE) + .takes_value(false) + .long(DISABLE_EXPECT_CRASH_ON_FAILURE), + ] +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("execute a local-only libfuzzer fuzzing task") + .args(&build_shared_args()) +} diff --git a/src/agent/onefuzz-agent/src/local/mod.rs b/src/agent/onefuzz-agent/src/local/mod.rs new file mode 100644 index 0000000000..7bb84bc571 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/mod.rs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +pub mod cmd; +pub mod common; +pub mod generic_crash_report; +pub mod generic_generator; +pub mod libfuzzer; +pub mod libfuzzer_coverage; +pub mod libfuzzer_crash_report; +pub mod libfuzzer_fuzz; +pub mod radamsa; diff --git a/src/agent/onefuzz-agent/src/local/radamsa.rs b/src/agent/onefuzz-agent/src/local/radamsa.rs new file mode 100644 index 0000000000..e7fda771f8 --- /dev/null +++ b/src/agent/onefuzz-agent/src/local/radamsa.rs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + local::{ + generic_crash_report::{build_report_config, build_shared_args as build_crash_args}, + generic_generator::{build_fuzz_config, build_shared_args as build_fuzz_args}, + }, + tasks::{fuzz::generator::GeneratorTask, report::generic::ReportTask}, +}; +use anyhow::Result; +use clap::{App, SubCommand}; +use std::collections::HashSet; +use tokio::task::spawn; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let fuzz_config = build_fuzz_config(args)?; + let fuzzer = GeneratorTask::new(fuzz_config); + let fuzz_task = spawn(async move { fuzzer.run().await }); + + let report_config = build_report_config(args)?; + let report = ReportTask::new(report_config); + let report_task = spawn(async move { report.local_run().await }); + + let result = tokio::try_join!(fuzz_task, report_task)?; + result.0?; + result.1?; + + Ok(()) +} + +pub fn args(name: &'static str) -> App<'static, 'static> { + let mut app = SubCommand::with_name(name).about("run a local generator & crash reporting job"); + + let mut used = HashSet::new(); + for args in &[build_fuzz_args(), build_crash_args()] { + for arg in args { + if used.contains(arg.b.name) { + continue; + } + used.insert(arg.b.name.to_string()); + app = app.arg(arg); + } + } + + app +} diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 52e101ed66..0ad9c284fe 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -10,19 +10,22 @@ extern crate onefuzz; #[macro_use] extern crate clap; -use std::path::PathBuf; - use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use onefuzz::telemetry::{self}; +use clap::{App, ArgMatches, SubCommand}; +use std::io::{stdout, Write}; mod debug; +mod local; +mod managed; mod tasks; -use tasks::config::Config; +const LICENSE_CMD: &str = "licenses"; +const LOCAL_CMD: &str = "local"; +const DEBUG_CMD: &str = "debug"; +const MANAGED_CMD: &str = "managed"; fn main() -> Result<()> { - env_logger::init(); + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); let built_version = format!( "{} onefuzz:{} git:{}", @@ -33,65 +36,30 @@ fn main() -> Result<()> { let app = App::new("onefuzz-agent") .version(built_version.as_str()) - .arg( - Arg::with_name("config") - .long("config") - .short("c") - .takes_value(true), - ) - .arg( - Arg::with_name("setup_dir") - .long("setup_dir") - .short("s") - .takes_value(true), - ) - .subcommand(debug::cmd::args()) - .subcommand(SubCommand::with_name("licenses").about("display third-party licenses")); + .subcommand(managed::cmd::args(MANAGED_CMD)) + .subcommand(local::cmd::args(LOCAL_CMD)) + .subcommand(debug::cmd::args(DEBUG_CMD)) + .subcommand(SubCommand::with_name(LICENSE_CMD).about("display third-party licenses")); let matches = app.get_matches(); - match matches.subcommand() { - ("licenses", Some(_)) => { - return licenses(); - } - ("debug", Some(sub)) => return crate::debug::cmd::run(sub), - _ => {} // no subcommand - } - - if matches.value_of("config").is_none() { - println!("Missing '--config'\n{}", matches.usage()); - return Ok(()); - } - - let config_path: PathBuf = matches.value_of("config").unwrap().parse()?; - let setup_dir = matches.value_of("setup_dir"); - let config = Config::from_file(config_path, setup_dir)?; - - init_telemetry(&config); - - verbose!("config parsed"); - let mut rt = tokio::runtime::Runtime::new()?; + rt.block_on(run(matches)) +} - let result = rt.block_on(config.run()); - - if let Err(err) = &result { - error!("error running task: {}", err); +async fn run(args: ArgMatches<'_>) -> Result<()> { + match args.subcommand() { + (LICENSE_CMD, Some(_)) => return licenses(), + (DEBUG_CMD, Some(sub)) => return debug::cmd::run(sub).await, + (LOCAL_CMD, Some(sub)) => return local::cmd::run(sub).await, + (MANAGED_CMD, Some(sub)) => return managed::cmd::run(sub).await, + _ => { + anyhow::bail!("missing subcommand\nUSAGE: {}", args.usage()); + } } - - telemetry::try_flush_and_close(); - - result } fn licenses() -> Result<()> { - use std::io::{self, Write}; - io::stdout().write_all(include_bytes!("../../data/licenses.json"))?; + stdout().write_all(include_bytes!("../../data/licenses.json"))?; Ok(()) } - -fn init_telemetry(config: &Config) { - let inst_key = config.common().instrumentation_key; - let tele_key = config.common().telemetry_key; - telemetry::set_appinsights_clients(inst_key, tele_key); -} diff --git a/src/agent/onefuzz-agent/src/managed/cmd.rs b/src/agent/onefuzz-agent/src/managed/cmd.rs new file mode 100644 index 0000000000..4cd49b41b7 --- /dev/null +++ b/src/agent/onefuzz-agent/src/managed/cmd.rs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::tasks::config::{CommonConfig, Config}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use onefuzz::telemetry; +use std::path::PathBuf; + +pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { + let config_path = value_t!(args, "config", PathBuf)?; + let setup_dir = value_t!(args, "setup_dir", PathBuf)?; + let config = Config::from_file(config_path, setup_dir)?; + + init_telemetry(config.common()); + let result = config.run().await; + + if let Err(err) = &result { + error!("error running task: {}", err); + } + + telemetry::try_flush_and_close(); + result +} + +fn init_telemetry(config: &CommonConfig) { + telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key); +} + +pub fn args(name: &str) -> App<'static, 'static> { + SubCommand::with_name(name) + .about("managed fuzzing") + .arg(Arg::with_name("config").required(true)) + .arg(Arg::with_name("setup_dir").required(true)) +} diff --git a/src/agent/onefuzz-agent/src/managed/mod.rs b/src/agent/onefuzz-agent/src/managed/mod.rs new file mode 100644 index 0000000000..52958ec91f --- /dev/null +++ b/src/agent/onefuzz-agent/src/managed/mod.rs @@ -0,0 +1 @@ +pub mod cmd; diff --git a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs index 6cce9171fa..2cc4b4ecdb 100644 --- a/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/analysis/generic.rs @@ -66,8 +66,9 @@ async fn run_existing(config: &Config) -> Result<()> { async fn already_checked(config: &Config, input: &BlobUrl) -> Result { let result = if let Some(crashes) = &config.crashes { - crashes.url.account() == input.account() - && crashes.url.container() == input.container() + let url = crashes.try_url()?; + url.account() == input.account() + && url.container() == input.container() && crashes.path.join(input.name()).exists() } else { false diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 02dc90b1d8..9849e9a56b 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -10,7 +10,7 @@ use onefuzz::{ }; use reqwest::Url; use serde::{self, Deserialize}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use uuid::Uuid; @@ -20,7 +20,7 @@ pub enum ContainerType { Inputs, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Clone, Default)] pub struct CommonConfig { pub job_id: Uuid, @@ -34,6 +34,7 @@ pub struct CommonConfig { pub telemetry_key: Option, + #[serde(default)] pub setup_dir: PathBuf, } @@ -68,7 +69,7 @@ pub enum Config { GenericAnalysis(analysis::generic::Config), #[serde(alias = "generic_generator")] - GenericGenerator(fuzz::generator::GeneratorConfig), + GenericGenerator(fuzz::generator::Config), #[serde(alias = "generic_supervisor")] GenericSupervisor(fuzz::supervisor::SupervisorConfig), @@ -81,16 +82,29 @@ pub enum Config { } impl Config { - pub fn from_file(path: impl AsRef, setup_dir: Option>) -> Result { + pub fn from_file(path: PathBuf, setup_dir: PathBuf) -> Result { let json = std::fs::read_to_string(path)?; - let mut json_config: serde_json::Value = serde_json::from_str(&json)?; + let json_config: serde_json::Value = serde_json::from_str(&json)?; + // override the setup_dir in the config file with the parameter value if specified - if let Some(setup_dir) = setup_dir { - json_config["setup_dir"] = - serde_json::Value::String(setup_dir.as_ref().to_string_lossy().into()); - } + let mut config: Self = serde_json::from_value(json_config)?; + config.common_mut().setup_dir = setup_dir; + + Ok(config) + } - Ok(serde_json::from_value(json_config)?) + fn common_mut(&mut self) -> &mut CommonConfig { + match self { + Config::LibFuzzerFuzz(c) => &mut c.common, + Config::LibFuzzerMerge(c) => &mut c.common, + Config::LibFuzzerReport(c) => &mut c.common, + Config::LibFuzzerCoverage(c) => &mut c.common, + Config::GenericAnalysis(c) => &mut c.common, + Config::GenericMerge(c) => &mut c.common, + Config::GenericReport(c) => &mut c.common, + Config::GenericSupervisor(c) => &mut c.common, + Config::GenericGenerator(c) => &mut c.common, + } } pub fn common(&self) -> &CommonConfig { @@ -150,25 +164,29 @@ impl Config { match self { Config::LibFuzzerFuzz(config) => { fuzz::libfuzzer_fuzz::LibFuzzerFuzzTask::new(config)? - .start() + .run() .await } Config::LibFuzzerReport(config) => { report::libfuzzer_report::ReportTask::new(config) - .run() + .managed_run() .await } Config::LibFuzzerCoverage(config) => { - coverage::libfuzzer_coverage::CoverageTask::new(Arc::new(config)) - .run() + coverage::libfuzzer_coverage::CoverageTask::new(config) + .managed_run() .await } Config::LibFuzzerMerge(config) => merge::libfuzzer_merge::spawn(Arc::new(config)).await, Config::GenericAnalysis(config) => analysis::generic::spawn(config).await, - Config::GenericGenerator(config) => fuzz::generator::spawn(Arc::new(config)).await, + Config::GenericGenerator(config) => { + fuzz::generator::GeneratorTask::new(config).run().await + } Config::GenericSupervisor(config) => fuzz::supervisor::spawn(config).await, Config::GenericMerge(config) => merge::generic::spawn(Arc::new(config)).await, - Config::GenericReport(config) => report::generic::ReportTask::new(&config).run().await, + Config::GenericReport(config) => { + report::generic::ReportTask::new(config).managed_run().await + } } } } diff --git a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs index 69cf35754b..2d1c287a2c 100644 --- a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs @@ -65,6 +65,9 @@ pub struct Config { pub readonly_inputs: Vec, pub coverage: SyncedDir, + #[serde(default = "default_bool_true")] + pub check_queue: bool, + #[serde(default = "default_bool_true")] pub check_fuzzer_help: bool, @@ -86,17 +89,26 @@ pub struct CoverageTask { } impl CoverageTask { - pub fn new(config: impl Into>) -> Self { - let config = config.into(); + pub fn new(config: Config) -> Self { + let config = Arc::new(config); + let poller = InputPoller::new(); + Self { config, poller } + } - let task_dir = PathBuf::from(config.common.task_id.to_string()); - let poller_dir = task_dir.join("poller"); - let poller = InputPoller::::new(poller_dir); + pub async fn local_run(&self) -> Result<()> { + let mut processor = CoverageProcessor::new(self.config.clone()).await?; - Self { config, poller } + self.config.coverage.init().await?; + for synced_dir in &self.config.readonly_inputs { + synced_dir.init().await?; + self.record_corpus_coverage(&mut processor, &synced_dir) + .await?; + } + + Ok(()) } - pub async fn run(&mut self) -> Result<()> { + pub async fn managed_run(&mut self) -> Result<()> { info!("starting libFuzzer coverage task"); if self.config.check_fuzzer_help { @@ -116,6 +128,7 @@ impl CoverageTask { async fn process(&mut self) -> Result<()> { let mut processor = CoverageProcessor::new(self.config.clone()).await?; + info!("processing initial dataset"); let mut seen_inputs = false; // Update the total with the coverage from each seed corpus. for dir in &self.config.readonly_inputs { @@ -144,7 +157,7 @@ impl CoverageTask { // If a queue has been provided, poll it for new coverage. if let Some(queue) = &self.config.input_queue { - verbose!("polling queue for new coverage"); + info!("polling queue for new coverage"); let callback = CallbackImpl::new(queue.clone(), processor); self.poller.run(callback).await?; } @@ -273,7 +286,7 @@ impl CoverageProcessor { #[async_trait] impl Processor for CoverageProcessor { - async fn process(&mut self, _url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, _url: Option, input: &Path) -> Result<()> { self.heartbeat_client.alive(); self.test_input(input).await?; self.report_total().await?; diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index 260525c376..140d02bfc7 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -6,7 +6,7 @@ use crate::tasks::{ heartbeat::*, utils::{self, default_bool_true}, }; -use anyhow::{Error, Result}; +use anyhow::Result; use futures::stream::StreamExt; use onefuzz::{ expand::Expand, @@ -23,18 +23,18 @@ use std::{ ffi::OsString, path::{Path, PathBuf}, process::Stdio, - sync::Arc, }; +use tempfile::tempdir; use tokio::{fs, process::Command}; #[derive(Debug, Deserialize, Clone)] -pub struct GeneratorConfig { +pub struct Config { pub generator_exe: String, pub generator_env: HashMap, pub generator_options: Vec, pub readonly_inputs: Vec, pub crashes: SyncedDir, - pub tools: SyncedDir, + pub tools: Option, pub target_exe: PathBuf, pub target_env: HashMap, @@ -52,134 +52,141 @@ pub struct GeneratorConfig { pub common: CommonConfig, } -pub async fn spawn(config: Arc) -> Result<(), Error> { - config.crashes.init().await?; - config.tools.init_pull().await?; +pub struct GeneratorTask { + config: Config, +} + +impl GeneratorTask { + pub fn new(config: Config) -> Self { + Self { config } + } + + pub async fn run(&self) -> Result<()> { + self.config.crashes.init().await?; + if let Some(tools) = &self.config.tools { + if tools.url.is_some() { + tools.init_pull().await?; + set_executable(&tools.path).await?; + } + } + + let hb_client = self.config.common.init_heartbeat().await?; + + for dir in &self.config.readonly_inputs { + dir.init_pull().await?; + } - set_executable(&config.tools.path).await?; - let hb_client = config.common.init_heartbeat().await?; + let sync_task = continuous_sync( + &self.config.readonly_inputs, + Pull, + self.config.ensemble_sync_delay, + ); - for dir in &config.readonly_inputs { - dir.init_pull().await?; + let crash_dir_monitor = self.config.crashes.monitor_results(new_result); + + let fuzzer = self.fuzzing_loop(hb_client); + + futures::try_join!(fuzzer, sync_task, crash_dir_monitor)?; + Ok(()) } - let sync_task = continuous_sync(&config.readonly_inputs, Pull, config.ensemble_sync_delay); - let crash_dir_monitor = config.crashes.monitor_results(new_result); - let tester = Tester::new( - &config.common.setup_dir, - &config.target_exe, - &config.target_options, - &config.target_env, - &config.target_timeout, - config.check_asan_log, - false, - config.check_debugger, - config.check_retry_count, - ); - let inputs: Vec<_> = config.readonly_inputs.iter().map(|x| &x.path).collect(); - let fuzzing_monitor = start_fuzzing(&config, inputs, tester, hb_client); - futures::try_join!(fuzzing_monitor, sync_task, crash_dir_monitor)?; - Ok(()) -} + async fn fuzzing_loop(&self, heartbeat_client: Option) -> Result<()> { + let tester = Tester::new( + &self.config.common.setup_dir, + &self.config.target_exe, + &self.config.target_options, + &self.config.target_env, + &self.config.target_timeout, + self.config.check_asan_log, + false, + self.config.check_debugger, + self.config.check_retry_count, + ); + + loop { + for corpus_dir in &self.config.readonly_inputs { + heartbeat_client.alive(); + let corpus_dir = &corpus_dir.path; + let generated_inputs = tempdir()?; + let generated_inputs_path = generated_inputs.path(); -async fn generate_input( - generator_exe: &str, - generator_env: &HashMap, - generator_options: &[String], - tools_dir: impl AsRef, - corpus_dir: impl AsRef, - output_dir: impl AsRef, -) -> Result<()> { - let mut expand = Expand::new(); - expand - .generated_inputs(&output_dir) - .input_corpus(&corpus_dir) - .generator_exe(&generator_exe) - .generator_options(&generator_options) - .tools_dir(&tools_dir); - - utils::reset_tmp_dir(&output_dir).await?; - - let generator_path = Expand::new() - .tools_dir(tools_dir.as_ref()) - .evaluate_value(generator_exe)?; - - let mut generator = Command::new(&generator_path); - generator - .kill_on_drop(true) - .env_remove("RUST_LOG") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - for arg in expand.evaluate(generator_options)? { - generator.arg(arg); + self.generate_inputs(corpus_dir, &generated_inputs_path) + .await?; + self.test_inputs(&generated_inputs_path, &tester).await?; + } + } } - for (k, v) in generator_env { - generator.env(k, expand.evaluate_value(v)?); + async fn test_inputs( + &self, + generated_inputs: impl AsRef, + tester: &Tester<'_>, + ) -> Result<()> { + let mut read_dir = fs::read_dir(generated_inputs).await?; + while let Some(file) = read_dir.next().await { + let file = file?; + + verbose!("testing input: {:?}", file); + + let destination_file = if self.config.rename_output { + let hash = sha256::digest_file(file.path()).await?; + OsString::from(hash) + } else { + file.file_name() + }; + + let destination_file = self.config.crashes.path.join(destination_file); + if tester.is_crash(file.path()).await? { + fs::rename(file.path(), &destination_file).await?; + verbose!("crash found {}", destination_file.display()); + } + } + Ok(()) } - info!("Generating test cases with {:?}", generator); - let output = generator.spawn()?; - monitor_process(output, "generator".to_string(), true, None).await?; + async fn generate_inputs( + &self, + corpus_dir: impl AsRef, + output_dir: impl AsRef, + ) -> Result<()> { + utils::reset_tmp_dir(&output_dir).await?; + let mut generator = { + let mut expand = Expand::new(); + expand + .generated_inputs(&output_dir) + .input_corpus(&corpus_dir) + .generator_exe(&self.config.generator_exe) + .generator_options(&self.config.generator_options); - Ok(()) -} + if let Some(tools) = &self.config.tools { + expand.tools_dir(&tools.path); + } -async fn start_fuzzing<'a>( - config: &GeneratorConfig, - corpus_dirs: Vec>, - tester: Tester<'_>, - heartbeat_client: Option, -) -> Result<()> { - let generator_tmp = "generator_tmp"; - - info!("Starting generator fuzzing loop"); - - loop { - heartbeat_client.alive(); - - for corpus_dir in &corpus_dirs { - let corpus_dir = corpus_dir.as_ref(); - - generate_input( - &config.generator_exe, - &config.generator_env, - &config.generator_options, - &config.tools.path, - corpus_dir, - generator_tmp, - ) - .await?; + let generator_path = expand.evaluate_value(&self.config.generator_exe)?; - let mut read_dir = fs::read_dir(generator_tmp).await?; - while let Some(file) = read_dir.next().await { - verbose!("Processing file {:?}", file); - let file = file?; - - let destination_file = if config.rename_output { - let hash = sha256::digest_file(file.path()).await?; - OsString::from(hash) - } else { - file.file_name() - }; - - let destination_file = config.crashes.path.join(destination_file); - if tester.is_crash(file.path()).await? { - info!("Crash found, path = {}", file.path().display()); - - if let Err(err) = fs::rename(file.path(), &destination_file).await { - warn!("Unable to move file {:?} : {:?}", file.path(), err); - } - } + let mut generator = Command::new(&generator_path); + generator + .kill_on_drop(true) + .env_remove("RUST_LOG") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + for arg in expand.evaluate(&self.config.generator_options)? { + generator.arg(arg); } - verbose!( - "Tested generated inputs for corpus = {}", - corpus_dir.display() - ); - } + for (k, v) in &self.config.generator_env { + generator.env(k, expand.evaluate_value(v)?); + } + generator + }; + + info!("Generating test cases with {:?}", generator); + let output = generator.spawn()?; + monitor_process(output, "generator".to_string(), true, None).await?; + + Ok(()) } } @@ -187,16 +194,22 @@ mod tests { #[tokio::test] #[cfg(target_os = "linux")] #[ignore] - async fn test_radamsa_linux() { - use super::*; + async fn test_radamsa_linux() -> anyhow::Result<()> { + use super::{Config, GeneratorTask}; + use crate::tasks::config::CommonConfig; + use onefuzz::syncdir::SyncedDir; + use std::collections::HashMap; use std::env; + use std::path::Path; + use tempfile::tempdir; + + let crashes_temp = tempfile::tempdir()?; + let crashes = crashes_temp.path(); - let radamsa_path = env::var("ONEFUZZ_TEST_RADAMSA_LINUX").unwrap(); - let corpus_dir_temp = tempfile::tempdir().unwrap(); - let corpus_dir = corpus_dir_temp.into_path(); - let seed_file_name = corpus_dir.clone().join("seed.txt"); - let radamsa_output_temp = tempfile::tempdir().unwrap(); - let radamsa_output = radamsa_output_temp.into_path(); + let inputs_temp = tempfile::tempdir().unwrap(); + let inputs = inputs_temp.path(); + let input_file = inputs.join("seed.txt"); + tokio::fs::write(input_file, "test").await?; let generator_options: Vec = vec![ "-o", @@ -210,22 +223,45 @@ mod tests { .map(|p| p.to_string()) .collect(); + let radamsa_path = env::var("ONEFUZZ_TEST_RADAMSA_LINUX")?; let radamsa_as_path = Path::new(&radamsa_path); let radamsa_dir = radamsa_as_path.parent().unwrap(); - let radamsa_exe = String::from("{tools_dir}/radamsa"); - let radamsa_env = HashMap::new(); - - tokio::fs::write(seed_file_name, "test").await.unwrap(); - let _output = generate_input( - &radamsa_exe, - &radamsa_env, - &generator_options, - &radamsa_dir, - corpus_dir, - radamsa_output.clone(), - ) - .await; - let generated_outputs = std::fs::read_dir(radamsa_output.clone()).unwrap(); - assert_eq!(generated_outputs.count(), 100, "No crashes generated"); + + let config = Config { + generator_exe: String::from("{tools_dir}/radamsa"), + generator_options, + readonly_inputs: vec![SyncedDir { + path: inputs.to_path_buf(), + url: None, + }], + crashes: SyncedDir { + path: crashes.to_path_buf(), + url: None, + }, + tools: Some(SyncedDir { + path: radamsa_dir.to_path_buf(), + url: None, + }), + target_exe: Default::default(), + target_env: Default::default(), + target_options: Default::default(), + target_timeout: None, + check_asan_log: false, + check_debugger: false, + rename_output: false, + ensemble_sync_delay: None, + generator_env: HashMap::default(), + check_retry_count: 0, + common: CommonConfig::default(), + }; + let task = GeneratorTask::new(config); + + let generated_inputs = tempdir()?; + task.generate_inputs(inputs.to_path_buf(), generated_inputs.path()) + .await?; + + let count = std::fs::read_dir(generated_inputs.path())?.count(); + assert_eq!(count, 100, "No inputs generated"); + Ok(()) } } diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index dbb28957b9..f7f652af3d 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -36,6 +36,11 @@ const PROC_INFO_PERIOD: Duration = Duration::from_secs(30); // Period of reporting fuzzer-generated runtime stats. const RUNTIME_STATS_PERIOD: Duration = Duration::from_secs(60); +pub fn default_workers() -> u64 { + let cpus = num_cpus::get() as u64; + u64::max(1, cpus - 1) +} + #[derive(Debug, Deserialize, Clone)] pub struct Config { pub inputs: SyncedDir, @@ -44,7 +49,9 @@ pub struct Config { pub target_exe: PathBuf, pub target_env: HashMap, pub target_options: Vec, - pub target_workers: Option, + + #[serde(default = "default_workers")] + pub target_workers: u64, pub ensemble_sync_delay: Option, #[serde(default = "default_bool_true")] @@ -66,23 +73,16 @@ impl LibFuzzerFuzzTask { Ok(Self { config }) } - pub async fn start(&self) -> Result<()> { - if self.config.check_fuzzer_help { - let target = LibFuzzer::new( - &self.config.target_exe, - &self.config.target_options, - &self.config.target_env, - &self.config.common.setup_dir, - ); - target.check_help().await?; + fn workers(&self) -> u64 { + match self.config.target_workers { + 0 => default_workers(), + x => x, } + } - let workers = self.config.target_workers.unwrap_or_else(|| { - let cpus = num_cpus::get() as u64; - u64::max(1, cpus - 1) - }); - + pub async fn run(&self) -> Result<()> { self.init_directories().await?; + let hb_client = self.config.common.init_heartbeat().await?; // To be scheduled. @@ -91,15 +91,19 @@ impl LibFuzzerFuzzTask { let new_crashes = self.config.crashes.monitor_results(new_result); let (stats_sender, stats_receiver) = mpsc::unbounded_channel(); - let report_stats = report_runtime_stats(workers as usize, stats_receiver, hb_client); + let report_stats = report_runtime_stats(self.workers() as usize, stats_receiver, hb_client); + let fuzzers = self.run_fuzzers(Some(&stats_sender)); + futures::try_join!(resync, new_inputs, new_crashes, fuzzers, report_stats)?; - let fuzzers: Vec<_> = (0..workers) - .map(|id| self.start_fuzzer_monitor(id, Some(&stats_sender))) - .collect(); + Ok(()) + } - let fuzzers = try_join_all(fuzzers); + pub async fn run_fuzzers(&self, stats_sender: Option<&StatsSender>) -> Result<()> { + let fuzzers: Vec<_> = (0..self.workers()) + .map(|id| self.start_fuzzer_monitor(id, stats_sender)) + .collect(); - futures::try_join!(resync, new_inputs, new_crashes, fuzzers, report_stats)?; + try_join_all(fuzzers).await?; Ok(()) } @@ -146,7 +150,7 @@ impl LibFuzzerFuzzTask { let crash_dir = tempdir()?; let run_id = Uuid::new_v4(); - info!("starting fuzzer run, run_id = {}", run_id); + verbose!("starting fuzzer run, run_id = {}", run_id); let mut inputs = vec![&self.config.inputs.path]; if let Some(readonly_inputs) = &self.config.readonly_inputs { diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs index bb1e48d018..5a01481bb3 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller.rs @@ -5,8 +5,9 @@ use std::{fmt, path::PathBuf}; use anyhow::Result; use futures::stream::StreamExt; -use onefuzz::{blob::BlobUrl, fs::OwnedDir, jitter::delay_with_jitter, syncdir::SyncedDir}; +use onefuzz::{blob::BlobUrl, jitter::delay_with_jitter, syncdir::SyncedDir}; use reqwest::Url; +use tempfile::{tempdir, TempDir}; use tokio::{fs, time::Duration}; mod callback; @@ -17,12 +18,12 @@ const POLL_INTERVAL: Duration = Duration::from_secs(10); #[cfg(test)] mod tests; -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Debug)] pub enum State { Ready, Polled(Option), Parsed(M, Url), - Downloaded(M, Url, PathBuf), + Downloaded(M, Url, PathBuf, TempDir), Processed(M), } @@ -78,10 +79,6 @@ impl<'a, M> fmt::Debug for Event<'a, M> { /// application data (here, the input URL, in some encoding) and metadata for /// operations like finalizing a dequeue with a pop receipt. pub struct InputPoller { - /// Agent-local directory where the poller will download inputs. - /// Will be reset for each new input. - working_dir: OwnedDir, - /// Internal automaton state. /// /// This is only nullable so we can internally `take()` the current state @@ -92,12 +89,10 @@ pub struct InputPoller { } impl InputPoller { - pub fn new(working_dir: impl Into) -> Self { - let working_dir = OwnedDir::new(working_dir); + pub fn new() -> Self { let state = Some(State::Ready); Self { state, - working_dir, batch_dir: None, } } @@ -109,11 +104,14 @@ impl InputPoller { to_process: &SyncedDir, ) -> Result<()> { self.batch_dir = Some(to_process.clone()); - to_process.init_pull().await?; + if to_process.url.is_some() { + to_process.init_pull().await?; + } + info!("batch processing directory: {}", to_process.path.display()); let mut read_dir = fs::read_dir(&to_process.path).await?; while let Some(file) = read_dir.next().await { - verbose!("Processing batch-downloaded input {:?}", file); + info!("Processing batch-downloaded input {:?}", file); let file = file?; let path = file.path(); @@ -126,7 +124,7 @@ impl InputPoller { let dir_relative = input_path.strip_prefix(&dir_path)?; dir_relative.display().to_string() }; - let url = to_process.url.blob(blob_name).url(); + let url = to_process.try_url().map(|x| x.blob(blob_name).url()).ok(); processor.process(url, &path).await?; } @@ -137,8 +135,8 @@ impl InputPoller { pub async fn seen_in_batch(&self, url: &Url) -> Result { let result = if let Some(batch_dir) = &self.batch_dir { if let Ok(blob) = BlobUrl::new(url.clone()) { - batch_dir.url.account() == blob.account() - && batch_dir.url.container() == blob.container() + batch_dir.try_url()?.account() == blob.account() + && batch_dir.try_url()?.container() == blob.container() && batch_dir.path.join(blob.name()).exists() } else { false @@ -149,15 +147,6 @@ impl InputPoller { Ok(result) } - /// Path to the working directory. - /// - /// We will create or reset the working directory before entering the - /// `Downloaded` state, but a caller cannot otherwise assume it exists. - #[allow(unused)] - pub fn working_dir(&self) -> &OwnedDir { - &self.working_dir - } - /// Get the current automaton state, including the state data. pub fn state(&self) -> &State { self.state.as_ref().unwrap_or_else(|| unreachable!()) @@ -168,13 +157,14 @@ impl InputPoller { } pub async fn run(&mut self, mut cb: impl Callback) -> Result<()> { + info!("starting input queue polling"); loop { match self.state() { State::Polled(None) => { verbose!("Input queue empty, sleeping"); delay_with_jitter(POLL_INTERVAL).await; } - State::Downloaded(_msg, _url, input) => { + State::Downloaded(_msg, _url, input, _tempdir) => { info!("Processing downloaded input: {:?}", input); } _ => {} @@ -249,21 +239,23 @@ impl InputPoller { } } (Parsed(msg, url), Download(downloader)) => { - self.working_dir.reset().await?; - + let download_dir = tempdir()?; if self.seen_in_batch(&url).await? { verbose!("url was seen during batch processing: {:?}", url); self.set_state(Processed(msg)); } else { let input = downloader - .download(url.clone(), self.working_dir.path()) + .download(url.clone(), download_dir.path()) .await?; - self.set_state(Downloaded(msg, url, input)); + self.set_state(Downloaded(msg, url, input, download_dir)); } } - (Downloaded(msg, url, input), Process(processor)) => { - processor.process(url, &input).await?; + // NOTE: _download_dir is a TempDir, which the physical path gets + // deleted automatically upon going out of scope. Keep it in-scope until + // here. + (Downloaded(msg, url, input, _download_dir), Process(processor)) => { + processor.process(Some(url), &input).await?; self.set_state(Processed(msg)); } diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs index 552a04f7ec..8148e0d97c 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/callback.rs @@ -26,7 +26,7 @@ pub trait Downloader { #[async_trait] pub trait Processor { - async fn process(&mut self, url: Url, input: &Path) -> Result<()>; + async fn process(&mut self, url: Option, input: &Path) -> Result<()>; } pub trait Callback { diff --git a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs index f80cbb6a52..464e644324 100644 --- a/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs +++ b/src/agent/onefuzz-agent/src/tasks/generic/input_poller/tests.rs @@ -5,7 +5,6 @@ use anyhow::Result; use async_trait::async_trait; use reqwest::Url; use std::path::Path; -use tempfile::{tempdir, TempDir}; use super::*; @@ -84,12 +83,12 @@ impl Downloader for TestDownloader { #[derive(Default)] struct TestProcessor { - processed: Vec<(Url, PathBuf)>, + processed: Vec<(Option, PathBuf)>, } #[async_trait] impl Processor for TestProcessor { - async fn process(&mut self, url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, url: Option, input: &Path) -> Result<()> { self.processed.push((url, input.to_owned())); Ok(()) @@ -100,11 +99,10 @@ fn url_input_name(url: &Url) -> String { url.path_segments().unwrap().last().unwrap().to_owned() } -fn fixture() -> (TempDir, InputPoller) { - let dir = tempdir().unwrap(); - let task = InputPoller::new(dir.path()); +fn fixture() -> InputPoller { + let task = InputPoller::new(); - (dir, task) + task } fn url_fixture(msg: Msg) -> Url { @@ -118,7 +116,7 @@ fn input_fixture(dir: &Path, msg: Msg) -> PathBuf { #[tokio::test] async fn test_ready_poll() { - let (_, mut task) = fixture(); + let mut task = fixture(); let msg: Msg = 0; @@ -135,7 +133,7 @@ async fn test_ready_poll() { #[tokio::test] async fn test_polled_some_parse() { - let (_, mut task) = fixture(); + let mut task = fixture(); let msg: Msg = 0; let url = url_fixture(msg); @@ -153,7 +151,7 @@ async fn test_polled_some_parse() { #[tokio::test] async fn test_polled_none_parse() { - let (_, mut task) = fixture(); + let mut task = fixture(); task.set_state(State::Polled(None)); @@ -166,11 +164,12 @@ async fn test_polled_none_parse() { #[tokio::test] async fn test_parsed_download() { - let (dir, mut task) = fixture(); + let mut task = fixture(); + let dir = Path::new("etc"); let msg: Msg = 0; let url = url_fixture(msg); - let input = input_fixture(dir.path(), msg); + let input = input_fixture(&dir, msg); task.set_state(State::Parsed(msg, url.clone())); @@ -180,17 +179,27 @@ async fn test_parsed_download() { .await .unwrap(); - assert_eq!(task.state(), &State::Downloaded(msg, url.clone(), input)); - assert_eq!(downloader.downloaded, vec![url]); + match task.state() { + State::Downloaded(got_msg, got_url, got_path) => { + assert_eq!(*got_msg, msg); + assert_eq!(*got_url, url); + assert_eq!(got_path.file_name(), input.file_name()); + } + _ => { + panic!("unexpected state"); + } + } } #[tokio::test] async fn test_downloaded_process() { - let (dir, mut task) = fixture(); + let mut task = fixture(); + + let dir = Path::new("etc"); let msg: Msg = 0; let url = url_fixture(msg); - let input = input_fixture(dir.path(), msg); + let input = input_fixture(dir, msg); task.set_state(State::Downloaded(msg, url.clone(), input.clone())); @@ -199,12 +208,12 @@ async fn test_downloaded_process() { task.trigger(Event::Process(&mut processor)).await.unwrap(); assert_eq!(task.state(), &State::Processed(msg)); - assert_eq!(processor.processed, vec![(url, input)]); + assert_eq!(processor.processed, vec![(Some(url), input)]); } #[tokio::test] async fn test_processed_finish() { - let (_, mut task) = fixture(); + let mut task = fixture(); let msg: Msg = 0; @@ -220,7 +229,7 @@ async fn test_processed_finish() { #[tokio::test] async fn test_invalid_trigger() { - let (_, mut task) = fixture(); + let mut task = fixture(); let mut queue = TestQueue::default(); @@ -233,7 +242,7 @@ async fn test_invalid_trigger() { #[tokio::test] async fn test_valid_trigger_failed_action() { - let (_, mut task) = fixture(); + let mut task = fixture(); let mut queue = TestQueueAlwaysFails; diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index 184368cbd3..cdc39df792 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -4,22 +4,27 @@ use anyhow::Result; use onefuzz::{ asan::AsanLog, - blob::{BlobClient, BlobContainerUrl, BlobUrl}, + blob::{BlobClient, BlobUrl}, + fs::exists, syncdir::SyncedDir, - telemetry::Event::{new_report, new_unable_to_reproduce, new_unique_report}, + telemetry::{ + Event::{new_report, new_unable_to_reproduce, new_unique_report}, + EventData, + }, }; - -use reqwest::StatusCode; +use reqwest::{StatusCode, Url}; use reqwest_retry::SendRetry; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use tokio::fs; use uuid::Uuid; #[derive(Debug, Deserialize, Serialize)] pub struct CrashReport { pub input_sha256: String, - pub input_blob: InputBlob, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_blob: Option, pub executable: PathBuf, @@ -44,7 +49,8 @@ pub struct CrashReport { #[derive(Debug, Deserialize, Serialize)] pub struct NoCrash { pub input_sha256: String, - pub input_blob: InputBlob, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_blob: Option, pub executable: PathBuf, pub task_id: Uuid, pub job_id: Uuid, @@ -59,44 +65,44 @@ pub enum CrashTestResult { } // Conditionally upload a report, if it would not be a duplicate. -// -// Use SHA-256 of call stack as dedupe key. -async fn upload_deduped(report: &CrashReport, container: &BlobContainerUrl) -> Result<()> { +async fn upload(report: &T, url: Url) -> Result { let blob = BlobClient::new(); - let deduped_name = report.unique_blob_name(); - let deduped_url = container.blob(deduped_name).url(); let result = blob - .put(deduped_url) + .put(url) .json(report) // Conditional PUT, only if-not-exists. // https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations .header("If-None-Match", "*") .send_retry_default() .await?; - if result.status() == StatusCode::CREATED { - event!(new_unique_report;); - } - Ok(()) + Ok(result.status() == StatusCode::CREATED) } -async fn upload_report(report: &CrashReport, container: &BlobContainerUrl) -> Result<()> { - event!(new_report;); - let blob = BlobClient::new(); - let url = container.blob(report.blob_name()).url(); - blob.put(url).json(report).send_retry_default().await?; - Ok(()) -} - -async fn upload_no_repro(report: &NoCrash, container: &BlobContainerUrl) -> Result<()> { - event!(new_unable_to_reproduce;); - let blob = BlobClient::new(); - let url = container.blob(report.blob_name()).url(); - blob.put(url).json(report).send_retry_default().await?; - Ok(()) +async fn upload_or_save_local( + report: &T, + dest_name: &str, + container: &SyncedDir, +) -> Result { + match &container.url { + Some(blob_url) => { + let url = blob_url.blob(dest_name).url(); + upload(report, url).await + } + None => { + let path = container.path.join(dest_name); + if !exists(&path).await? { + let data = serde_json::to_vec(&report)?; + fs::write(path, data).await?; + Ok(true) + } else { + Ok(false) + } + } + } } impl CrashTestResult { - pub async fn upload( + pub async fn save( &self, unique_reports: &SyncedDir, reports: &Option, @@ -104,14 +110,26 @@ impl CrashTestResult { ) -> Result<()> { match self { Self::CrashReport(report) => { - upload_deduped(report, &unique_reports.url).await?; + // Use SHA-256 of call stack as dedupe key. + let name = report.unique_blob_name(); + if upload_or_save_local(&report, &name, unique_reports).await? { + event!(new_unique_report; EventData::Path = name); + } + if let Some(reports) = reports { - upload_report(report, &reports.url).await?; + let name = report.blob_name(); + if upload_or_save_local(&report, &name, reports).await? { + event!(new_report; EventData::Path = name); + } } } + Self::NoRepro(report) => { if let Some(no_repro) = no_repro { - upload_no_repro(report, &no_repro.url).await?; + let name = report.blob_name(); + if upload_or_save_local(&report, &name, no_repro).await? { + event!(new_unable_to_reproduce; EventData::Path = name); + } } } } @@ -142,7 +160,7 @@ impl CrashReport { task_id: Uuid, job_id: Uuid, executable: impl Into, - input_blob: InputBlob, + input_blob: Option, input_sha256: String, ) -> Self { Self { diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index 34170dfdd2..a69650b076 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -10,7 +10,10 @@ use crate::tasks::{ }; use anyhow::Result; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, input_tester::Tester, sha256, syncdir::SyncedDir}; +use futures::stream::StreamExt; +use onefuzz::{ + blob::BlobUrl, input_tester::Tester, monitor::DirectoryMonitor, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -44,35 +47,64 @@ pub struct Config { #[serde(default)] pub check_retry_count: u64, + #[serde(default = "default_bool_true")] + pub check_queue: bool, + #[serde(flatten)] pub common: CommonConfig, } -pub struct ReportTask<'a> { - config: &'a Config, +pub struct ReportTask { + config: Config, poller: InputPoller, } -impl<'a> ReportTask<'a> { - pub fn new(config: &'a Config) -> Self { - let working_dir = config.common.task_id.to_string(); - let poller = InputPoller::new(working_dir); - +impl ReportTask { + pub fn new(config: Config) -> Self { + let poller = InputPoller::new(); Self { config, poller } } - pub async fn run(&mut self) -> Result<()> { + pub async fn local_run(&self) -> Result<()> { + let mut processor = GenericReportProcessor::new(&self.config, None); + + info!("Starting generic crash report task"); + let crashes = match &self.config.crashes { + Some(x) => x, + None => bail!("missing crashes directory"), + }; + + let mut read_dir = tokio::fs::read_dir(&crashes.path).await?; + while let Some(crash) = read_dir.next().await { + processor.process(None, &crash?.path()).await?; + } + + if self.config.check_queue { + let mut monitor = DirectoryMonitor::new(&crashes.path); + monitor.start()?; + while let Some(crash) = monitor.next().await { + processor.process(None, &crash).await?; + } + } + Ok(()) + } + + pub async fn managed_run(&mut self) -> Result<()> { info!("Starting generic crash report task"); let heartbeat_client = self.config.common.init_heartbeat().await?; let mut processor = GenericReportProcessor::new(&self.config, heartbeat_client); + info!("processing existing crashes"); if let Some(crashes) = &self.config.crashes { self.poller.batch_process(&mut processor, &crashes).await?; } - if let Some(queue) = &self.config.input_queue { - let callback = CallbackImpl::new(queue.clone(), processor); - self.poller.run(callback).await?; + info!("processing crashes from queue"); + if self.config.check_queue { + if let Some(queue) = &self.config.input_queue { + let callback = CallbackImpl::new(queue.clone(), processor); + self.poller.run(callback).await?; + } } Ok(()) } @@ -105,12 +137,19 @@ impl<'a> GenericReportProcessor<'a> { } } - pub async fn test_input(&self, input_url: Url, input: &Path) -> Result { + pub async fn test_input( + &self, + input_url: Option, + input: &Path, + ) -> Result { self.heartbeat_client.alive(); let input_sha256 = sha256::digest_file(input).await?; let task_id = self.config.common.task_id; let job_id = self.config.common.job_id; - let input_blob = InputBlob::from(BlobUrl::new(input_url)?); + let input_blob = match input_url { + Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)), + None => None, + }; let test_report = self.tester.test_input(input).await?; @@ -160,10 +199,11 @@ impl<'a> GenericReportProcessor<'a> { #[async_trait] impl<'a> Processor for GenericReportProcessor<'a> { - async fn process(&mut self, url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, url: Option, input: &Path) -> Result<()> { + verbose!("generating crash report for: {}", input.display()); let report = self.test_input(url, input).await?; report - .upload( + .save( &self.config.unique_reports, &self.config.reports, &self.config.no_repro, diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index 175f6a3abb..9903c63576 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -5,9 +5,12 @@ use super::crash_report::*; use crate::tasks::{ config::CommonConfig, generic::input_poller::*, heartbeat::*, utils::default_bool_true, }; -use anyhow::Result; +use anyhow::{Context, Result}; use async_trait::async_trait; -use onefuzz::{blob::BlobUrl, libfuzzer::LibFuzzer, sha256, syncdir::SyncedDir}; +use futures::stream::StreamExt; +use onefuzz::{ + blob::BlobUrl, libfuzzer::LibFuzzer, monitor::DirectoryMonitor, sha256, syncdir::SyncedDir, +}; use reqwest::Url; use serde::Deserialize; use std::{ @@ -36,6 +39,9 @@ pub struct Config { #[serde(default)] pub check_retry_count: u64, + #[serde(default = "default_bool_true")] + pub check_queue: bool, + #[serde(flatten)] pub common: CommonConfig, } @@ -46,26 +52,52 @@ pub struct ReportTask { } impl ReportTask { - pub fn new(config: impl Into>) -> Self { - let config = config.into(); - - let working_dir = config.common.task_id.to_string(); - let poller = InputPoller::new(working_dir); + pub fn new(config: Config) -> Self { + let poller = InputPoller::new(); + let config = Arc::new(config); Self { config, poller } } - pub async fn run(&mut self) -> Result<()> { - if self.config.check_fuzzer_help { - let target = LibFuzzer::new( - &self.config.target_exe, - &self.config.target_options, - &self.config.target_env, - &self.config.common.setup_dir, - ); - target.check_help().await?; + pub async fn local_run(&self) -> Result<()> { + let mut processor = AsanProcessor::new(self.config.clone()).await?; + let crashes = match &self.config.crashes { + Some(x) => x, + None => bail!("missing crashes directory"), + }; + crashes.init().await?; + + self.config.unique_reports.init().await?; + if let Some(reports) = &self.config.reports { + reports.init().await?; } + if let Some(no_repro) = &self.config.no_repro { + no_repro.init().await?; + } + + let mut read_dir = tokio::fs::read_dir(&crashes.path).await.with_context(|| { + format_err!( + "unable to read crashes directory {}", + crashes.path.display() + ) + })?; + while let Some(crash) = read_dir.next().await { + processor.process(None, &crash?.path()).await?; + } + + if self.config.check_queue { + let mut monitor = DirectoryMonitor::new(crashes.path.clone()); + monitor.start()?; + while let Some(crash) = monitor.next().await { + processor.process(None, &crash).await?; + } + } + + Ok(()) + } + + pub async fn managed_run(&mut self) -> Result<()> { info!("Starting libFuzzer crash report task"); let mut processor = AsanProcessor::new(self.config.clone()).await?; @@ -73,9 +105,11 @@ impl ReportTask { self.poller.batch_process(&mut processor, crashes).await?; } - if let Some(queue) = &self.config.input_queue { - let callback = CallbackImpl::new(queue.clone(), processor); - self.poller.run(callback).await?; + if self.config.check_queue { + if let Some(queue) = &self.config.input_queue { + let callback = CallbackImpl::new(queue.clone(), processor); + self.poller.run(callback).await?; + } } Ok(()) } @@ -96,7 +130,11 @@ impl AsanProcessor { }) } - pub async fn test_input(&self, input_url: Url, input: &Path) -> Result { + pub async fn test_input( + &self, + input_url: Option, + input: &Path, + ) -> Result { self.heartbeat_client.alive(); let fuzzer = LibFuzzer::new( &self.config.target_exe, @@ -107,8 +145,13 @@ impl AsanProcessor { let task_id = self.config.common.task_id; let job_id = self.config.common.job_id; - let input_blob = InputBlob::from(BlobUrl::new(input_url)?); - let input_sha256 = sha256::digest_file(input).await?; + let input_blob = match input_url { + Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)), + None => None, + }; + let input_sha256 = sha256::digest_file(input).await.with_context(|| { + format_err!("unable to sha256 digest input file: {}", input.display()) + })?; let test_report = fuzzer .repro( @@ -149,10 +192,11 @@ impl AsanProcessor { #[async_trait] impl Processor for AsanProcessor { - async fn process(&mut self, url: Url, input: &Path) -> Result<()> { + async fn process(&mut self, url: Option, input: &Path) -> Result<()> { + verbose!("processing libfuzzer crash url:{:?} path:{:?}", url, input); let report = self.test_input(url, input).await?; report - .upload( + .save( &self.config.unique_reports, &self.config.reports, &self.config.no_repro, diff --git a/src/agent/onefuzz-supervisor/src/worker.rs b/src/agent/onefuzz-supervisor/src/worker.rs index c3e815773d..c3437c61c2 100644 --- a/src/agent/onefuzz-supervisor/src/worker.rs +++ b/src/agent/onefuzz-supervisor/src/worker.rs @@ -225,9 +225,8 @@ impl IWorkerRunner for WorkerRunner { let mut cmd = Command::new("onefuzz-agent"); cmd.current_dir(&working_dir); - cmd.arg("-c"); + cmd.arg("managed"); cmd.arg("config.json"); - cmd.arg("-s"); cmd.arg(setup_dir); cmd.stderr(Stdio::piped()); cmd.stdout(Stdio::piped()); diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index ae6cdf3da2..7ff1172589 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -16,6 +16,7 @@ use tokio::process::{Child, Command}; const DEFAULT_MAX_TOTAL_SECONDS: i32 = 10 * 60; +#[derive(Debug)] pub struct LibFuzzerMergeOutput { pub added_files_count: i32, pub added_feature_count: i32, diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index 3a41104cd8..1b9aa003ad 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -26,13 +26,18 @@ const DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS: u64 = 60; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct SyncedDir { pub path: PathBuf, - pub url: BlobContainerUrl, + pub url: Option, } impl SyncedDir { pub async fn sync(&self, operation: SyncOperation, delete_dst: bool) -> Result<()> { + if self.url.is_none() { + verbose!("not syncing as SyncedDir is missing remote URL"); + return Ok(()); + } + let dir = &self.path; - let url = self.url.url(); + let url = self.url.as_ref().unwrap().url(); let url = url.as_ref(); verbose!("syncing {:?} {}", operation, dir.display()); match operation { @@ -41,6 +46,14 @@ impl SyncedDir { } } + pub fn try_url(&self) -> Result<&BlobContainerUrl> { + let url = match &self.url { + Some(x) => x, + None => bail!("missing URL context"), + }; + Ok(url) + } + pub async fn init_pull(&self) -> Result<()> { self.init().await?; self.sync(SyncOperation::Pull, false).await @@ -55,7 +68,7 @@ impl SyncedDir { anyhow::bail!("File with name '{}' already exists", self.path.display()); } } - Err(_) => fs::create_dir(&self.path).await.with_context(|| { + Err(_) => fs::create_dir_all(&self.path).await.with_context(|| { format!("unable to create local SyncedDir: {}", self.path.display()) }), } @@ -74,6 +87,11 @@ impl SyncedDir { operation: SyncOperation, delay_seconds: Option, ) -> Result<()> { + if self.url.is_none() { + verbose!("not continuously syncing, as SyncDir does not have a remote URL"); + return Ok(()); + } + let delay_seconds = delay_seconds.unwrap_or(DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS); if delay_seconds == 0 { return Ok(()); @@ -86,23 +104,24 @@ impl SyncedDir { } } - async fn file_uploader_monitor(&self, event: Event) -> Result<()> { - let url = self.url.url(); + async fn file_monitor_event(&self, event: Event) -> Result<()> { verbose!("monitoring {}", self.path.display()); - let mut monitor = DirectoryMonitor::new(self.path.clone()); monitor.start()?; - let mut uploader = BlobUploader::new(url); + + let mut uploader = self.url.as_ref().map(|x| BlobUploader::new(x.url())); while let Some(item) = monitor.next().await { event!(event.clone(); EventData::Path = item.display().to_string()); - if let Err(err) = uploader.upload(item.clone()).await { - bail!( - "Couldn't upload file. path:{} dir:{} err:{}", - item.display(), - self.path.display(), - err - ); + if let Some(uploader) = &mut uploader { + if let Err(err) = uploader.upload(item.clone()).await { + bail!( + "Couldn't upload file. path:{} dir:{} err:{}", + item.display(), + self.path.display(), + err + ); + } } } @@ -129,16 +148,34 @@ impl SyncedDir { } verbose!("starting monitor for {}", self.path.display()); - self.file_uploader_monitor(event.clone()).await?; + self.file_monitor_event(event.clone()).await?; } } } +impl From for SyncedDir { + fn from(path: PathBuf) -> Self { + Self { path, url: None } + } +} + pub async fn continuous_sync( dirs: &[SyncedDir], operation: SyncOperation, delay_seconds: Option, ) -> Result<()> { + let mut should_loop = false; + for dir in dirs { + if dir.url.is_some() { + should_loop = true; + break; + } + } + if !should_loop { + verbose!("not syncing as SyncDirs do not have remote URLs"); + return Ok(()); + } + let delay_seconds = delay_seconds.unwrap_or(DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS); if delay_seconds == 0 { return Ok(()); diff --git a/src/agent/onefuzz/src/telemetry.rs b/src/agent/onefuzz/src/telemetry.rs index 66c955808e..f00d2818d7 100644 --- a/src/agent/onefuzz/src/telemetry.rs +++ b/src/agent/onefuzz/src/telemetry.rs @@ -309,6 +309,16 @@ pub fn set_property(entry: EventData) { } } +fn local_log_event(event: &Event, properties: &[EventData]) { + let as_values = properties + .iter() + .map(|x| x.as_values()) + .map(|(x, y)| format!("{}:{}", x, y)) + .collect::>() + .join(" "); + log::log!(log::Level::Info, "{} {}", event.as_str(), as_values); +} + pub fn track_event(event: Event, properties: Vec) { use appinsights::telemetry::Telemetry; @@ -334,6 +344,7 @@ pub fn track_event(event: Event, properties: Vec) { } client.track(evt); } + local_log_event(&event, &properties); } #[macro_export] diff --git a/src/api-service/__app__/onefuzzlib/tasks/scheduler.py b/src/api-service/__app__/onefuzzlib/tasks/scheduler.py index f46c46493a..94ed9b285e 100644 --- a/src/api-service/__app__/onefuzzlib/tasks/scheduler.py +++ b/src/api-service/__app__/onefuzzlib/tasks/scheduler.py @@ -154,7 +154,7 @@ def build_work_unit(task: Task) -> Optional[Tuple[BucketConfig, WorkUnit]]: job_id=task_config.job_id, task_id=task_config.task_id, task_type=task_config.task_type, - config=task_config.json(), + config=task_config.json(exclude_none=True, exclude_unset=True), ) bucket_config = BucketConfig( diff --git a/src/deployment/registration.py b/src/deployment/registration.py index 48b6636cc5..59cefa4932 100644 --- a/src/deployment/registration.py +++ b/src/deployment/registration.py @@ -13,6 +13,7 @@ from uuid import UUID, uuid4 import requests +from azure.cli.core.utils import get_az_rest_user_agent from azure.common.client_factory import get_client_from_cli_profile from azure.common.credentials import get_cli_profile from azure.graphrbac import GraphRbacManagementClient @@ -49,6 +50,7 @@ def query_microsoft_graph( headers = { "Authorization": "%s %s" % (token_type, access_token), "Content-Type": "application/json", + "User-Agent": get_az_rest_user_agent(), } response = requests.request( method=method, url=url, headers=headers, params=params, json=body