From fa062331dcf9d46898aa24f1c6b42f9443bef10d Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 10 Mar 2021 14:20:03 -0500 Subject: [PATCH 01/13] regression task suggestions --- .github/workflows/ci.yml | 4 + docs/webhook_events.md | 417 +++++++++++++++++- .../src/tasks/regression/common.rs | 165 +++---- .../src/tasks/regression/generic.rs | 55 +-- .../src/tasks/regression/libfuzzer.rs | 56 +-- .../src/tasks/report/crash_report.rs | 77 ++-- .../__app__/onefuzzlib/notifications/ado.py | 24 +- .../onefuzzlib/notifications/github_issues.py | 19 +- .../__app__/onefuzzlib/notifications/main.py | 20 +- .../__app__/onefuzzlib/notifications/teams.py | 11 +- src/api-service/__app__/onefuzzlib/reports.py | 41 +- .../__app__/onefuzzlib/tasks/defs.py | 40 +- src/api-service/tests/test_report_parse.py | 11 +- src/cli/onefuzz/templates/__init__.py | 6 + src/cli/onefuzz/templates/libfuzzer.py | 33 +- src/cli/onefuzz/templates/regression.py | 152 +++---- .../libfuzzer-regression/Makefile | 16 + .../libfuzzer-regression/seeds/good.txt | 1 + .../libfuzzer-regression/simple.c | 26 ++ src/pytypes/extra/generate-docs.py | 48 +- src/pytypes/onefuzztypes/enums.py | 5 +- src/pytypes/onefuzztypes/events.py | 19 +- src/pytypes/onefuzztypes/models.py | 22 +- 23 files changed, 923 insertions(+), 345 deletions(-) create mode 100644 src/integration-tests/libfuzzer-regression/Makefile create mode 100644 src/integration-tests/libfuzzer-regression/seeds/good.txt create mode 100644 src/integration-tests/libfuzzer-regression/simple.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4104e55667..4f6ae1cca7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,6 +340,10 @@ jobs: (cd libfuzzer ; make ) cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer + mkdir -p artifacts/linux-libfuzzer-regression + (cd libfuzzer-regression ; make ) + cp -r libfuzzer-regression/broken.exe libfuzzer-regression/fixed.exe libfuzzer-regression/seeds artifacts/linux-libfuzzer-regression + mkdir -p artifacts/linux-trivial-crash (cd trivial-crash ; make ) cp -r trivial-crash/fuzz.exe trivial-crash/seeds artifacts/linux-trivial-crash diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 6a3bc2f614..723ac62c95 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -37,6 +37,7 @@ Each event will be submitted via HTTP POST to the user provided URL. * [proxy_created](#proxy_created) * [proxy_deleted](#proxy_deleted) * [proxy_failed](#proxy_failed) +* [regression_reported](#regression_reported) * [scaleset_created](#scaleset_created) * [scaleset_deleted](#scaleset_deleted) * [scaleset_failed](#scaleset_failed) @@ -475,11 +476,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -1030,6 +1033,262 @@ Each event will be submitted via HTTP POST to the user provided URL. } ``` +### regression_reported + +#### Example + +```json +{ + "container": "container-name", + "filename": "example.json", + "regression_report": { + "crash_test_result": { + "crash_report": { + "asan_log": "example asan log", + "call_stack": [ + "#0 line", + "#1 line", + "#2 line" + ], + "call_stack_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "crash_site": "example crash site", + "crash_type": "example crash report type", + "executable": "fuzz.exe", + "input_blob": { + "account": "contoso-storage-account", + "container": "crashes", + "name": "input.txt" + }, + "input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "job_id": "00000000-0000-0000-0000-000000000000", + "scariness_description": "example-scariness", + "scariness_score": 10, + "task_id": "00000000-0000-0000-0000-000000000000" + } + }, + "original_crash_test_result": { + "crash_report": { + "asan_log": "example asan log", + "call_stack": [ + "#0 line", + "#1 line", + "#2 line" + ], + "call_stack_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "crash_site": "example crash site", + "crash_type": "example crash report type", + "executable": "fuzz.exe", + "input_blob": { + "account": "contoso-storage-account", + "container": "crashes", + "name": "input.txt" + }, + "input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "job_id": "00000000-0000-0000-0000-000000000000", + "scariness_description": "example-scariness", + "scariness_score": 10, + "task_id": "00000000-0000-0000-0000-000000000000" + } + } + } +} +``` + +#### Schema + +```json +{ + "additionalProperties": false, + "definitions": { + "BlobRef": { + "properties": { + "account": { + "title": "Account", + "type": "string" + }, + "container": { + "title": "Container", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "account", + "container", + "name" + ], + "title": "BlobRef", + "type": "object" + }, + "CrashTestResult": { + "properties": { + "crash_report": { + "$ref": "#/definitions/Report" + }, + "no_repro": { + "$ref": "#/definitions/NoReproReport" + } + }, + "title": "CrashTestResult", + "type": "object" + }, + "NoReproReport": { + "properties": { + "error": { + "title": "Error", + "type": "string" + }, + "executable": { + "title": "Executable", + "type": "string" + }, + "input_blob": { + "$ref": "#/definitions/BlobRef" + }, + "input_sha256": { + "title": "Input Sha256", + "type": "string" + }, + "job_id": { + "format": "uuid", + "title": "Job Id", + "type": "string" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "tries": { + "title": "Tries", + "type": "integer" + } + }, + "required": [ + "input_sha256", + "executable", + "task_id", + "job_id", + "tries" + ], + "title": "NoReproReport", + "type": "object" + }, + "RegressionReport": { + "properties": { + "crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + }, + "original_crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + } + }, + "required": [ + "crash_test_result" + ], + "title": "RegressionReport", + "type": "object" + }, + "Report": { + "properties": { + "asan_log": { + "title": "Asan Log", + "type": "string" + }, + "call_stack": { + "items": { + "type": "string" + }, + "title": "Call Stack", + "type": "array" + }, + "call_stack_sha256": { + "title": "Call Stack Sha256", + "type": "string" + }, + "crash_site": { + "title": "Crash Site", + "type": "string" + }, + "crash_type": { + "title": "Crash Type", + "type": "string" + }, + "executable": { + "title": "Executable", + "type": "string" + }, + "input_blob": { + "$ref": "#/definitions/BlobRef" + }, + "input_sha256": { + "title": "Input Sha256", + "type": "string" + }, + "input_url": { + "title": "Input Url", + "type": "string" + }, + "job_id": { + "format": "uuid", + "title": "Job Id", + "type": "string" + }, + "scariness_description": { + "title": "Scariness Description", + "type": "string" + }, + "scariness_score": { + "title": "Scariness Score", + "type": "integer" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + } + }, + "required": [ + "input_blob", + "executable", + "crash_type", + "crash_site", + "call_stack", + "call_stack_sha256", + "input_sha256", + "task_id", + "job_id" + ], + "title": "Report", + "type": "object" + } + }, + "properties": { + "container": { + "title": "Container", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "regression_report": { + "$ref": "#/definitions/RegressionReport" + } + }, + "required": [ + "regression_report", + "container", + "filename" + ], + "title": "EventRegressionReported", + "type": "object" +} +``` + ### scaleset_created #### Example @@ -1284,7 +1543,7 @@ Each event will be submitted via HTTP POST to the user provided URL. "tools", "unique_inputs", "unique_reports", - "input_reports" + "regression_reports" ], "title": "ContainerType" }, @@ -1725,7 +1984,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -1946,6 +2206,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -2044,11 +2311,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -2198,7 +2467,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -2371,6 +2641,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -2469,11 +2746,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -2597,7 +2876,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -2770,6 +3050,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -2882,11 +3169,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -3023,7 +3312,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -3196,6 +3486,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -3294,11 +3591,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -3479,10 +3778,22 @@ Each event will be submitted via HTTP POST to the user provided URL. "tools", "unique_inputs", "unique_reports", - "input_reports" + "regression_reports" ], "title": "ContainerType" }, + "CrashTestResult": { + "properties": { + "crash_report": { + "$ref": "#/definitions/Report" + }, + "no_repro": { + "$ref": "#/definitions/NoReproReport" + } + }, + "title": "CrashTestResult", + "type": "object" + }, "Error": { "properties": { "code": { @@ -3832,6 +4143,29 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "EventProxyFailed", "type": "object" }, + "EventRegressionReported": { + "additionalProperties": false, + "properties": { + "container": { + "title": "Container", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "regression_report": { + "$ref": "#/definitions/RegressionReport" + } + }, + "required": [ + "regression_report", + "container", + "filename" + ], + "title": "EventRegressionReported", + "type": "object" + }, "EventScalesetCreated": { "additionalProperties": false, "properties": { @@ -4085,6 +4419,7 @@ Each event will be submitted via HTTP POST to the user provided URL. "task_state_updated", "task_stopped", "crash_reported", + "regression_reported", "file_added", "task_heartbeat", "node_heartbeat" @@ -4140,6 +4475,48 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "JobTaskStopped", "type": "object" }, + "NoReproReport": { + "properties": { + "error": { + "title": "Error", + "type": "string" + }, + "executable": { + "title": "Executable", + "type": "string" + }, + "input_blob": { + "$ref": "#/definitions/BlobRef" + }, + "input_sha256": { + "title": "Input Sha256", + "type": "string" + }, + "job_id": { + "format": "uuid", + "title": "Job Id", + "type": "string" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "tries": { + "title": "Tries", + "type": "integer" + } + }, + "required": [ + "input_sha256", + "executable", + "task_id", + "job_id", + "tries" + ], + "title": "NoReproReport", + "type": "object" + }, "NodeState": { "description": "An enumeration.", "enum": [ @@ -4163,6 +4540,21 @@ Each event will be submitted via HTTP POST to the user provided URL. ], "title": "OS" }, + "RegressionReport": { + "properties": { + "crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + }, + "original_crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + } + }, + "required": [ + "crash_test_result" + ], + "title": "RegressionReport", + "type": "object" + }, "Report": { "properties": { "asan_log": { @@ -4658,6 +5050,9 @@ Each event will be submitted via HTTP POST to the user provided URL. { "$ref": "#/definitions/EventCrashReported" }, + { + "$ref": "#/definitions/EventRegressionReported" + }, { "$ref": "#/definitions/EventFileAdded" } diff --git a/src/agent/onefuzz-agent/src/tasks/regression/common.rs b/src/agent/onefuzz-agent/src/tasks/regression/common.rs index 3d7d19b19b..e75af1e57b 100644 --- a/src/agent/onefuzz-agent/src/tasks/regression/common.rs +++ b/src/agent/onefuzz-agent/src/tasks/regression/common.rs @@ -3,10 +3,9 @@ use crate::tasks::{ heartbeat::{HeartbeatSender, TaskHeartbeatClient}, - report::crash_report::{CrashReport, CrashTestResult}, - utils::download_input, + report::crash_report::{parse_report_file, CrashTestResult, RegressionReport}, }; -use anyhow::Result; +use anyhow::{Context, Result}; use async_trait::async_trait; use onefuzz::syncdir::SyncedDir; use reqwest::Url; @@ -23,117 +22,135 @@ pub trait RegressionHandler { input: PathBuf, input_url: Option, ) -> Result; - - /// Saves a regression - /// * `crash_result` - crash result to save - /// * `original_report` - original report used to generate the report - async fn save_regression( - &self, - crash_result: CrashTestResult, - original_report: Option, - ) -> Result<()>; } /// Runs the regression task -/// * `heartbeat_client` - heartbeat client -/// * `input_reports` - location of the reports used in this regression run -/// * `report_list` - list of report file names selected to be used in the regression -/// * `crashes` - location of the crash files referenced by the reports in input_reports -/// * `inputs` - location of the input files -/// * `handler` - regression handler pub async fn run( heartbeat_client: Option, - input_reports: &Option, - report_list: &[String], - crashes: &Option, - inputs: &Option, + regression_reports: &SyncedDir, + crashes: &SyncedDir, + report_dirs: &[&SyncedDir], + report_list: &Option>, + readonly_inputs: &Option, handler: &impl RegressionHandler, ) -> Result<()> { info!("Starting generic regression task"); - if let (Some(input_reports), Some(crashes)) = (input_reports, crashes) { - handle_crash_reports( - &heartbeat_client, - &input_reports, - report_list, - &crashes, + handle_crash_reports( + handler, + crashes, + report_dirs, + report_list, + ®ression_reports, + &heartbeat_client, + ) + .await?; + + if let Some(readonly_inputs) = &readonly_inputs { + handle_inputs( handler, + readonly_inputs, + ®ression_reports, + &heartbeat_client, ) .await?; } - if let Some(inputs) = &inputs { - handle_inputs(&inputs, &heartbeat_client, handler).await?; - } - Ok(()) } /// Run the regression on the files in the 'inputs' location -/// * `heartbeat_client` - heartbeat client -/// * `inputs` - location of the input files /// * `handler` - regression handler +/// * `readonly_inputs` - location of the input files +/// * `regression_reports` - where reports should be saved +/// * `heartbeat_client` - heartbeat client pub async fn handle_inputs( - inputs: &SyncedDir, - heartbeat_client: &Option, handler: &impl RegressionHandler, + readonly_inputs: &SyncedDir, + regression_reports: &SyncedDir, + heartbeat_client: &Option, ) -> Result<()> { - inputs.sync_pull().await?; - let mut input_files = tokio::fs::read_dir(&inputs.path).await?; + readonly_inputs.sync_pull().await?; + let mut input_files = tokio::fs::read_dir(&readonly_inputs.path).await?; while let Some(file) = input_files.next_entry().await? { heartbeat_client.alive(); - let input_url = inputs.url.clone().and_then(|container_url| { + let input_url = readonly_inputs.url.clone().and_then(|container_url| { let os_file_name = file.file_name(); let file_name = os_file_name.to_str()?; container_url.url().join(file_name).ok() }); - let report = handler.get_crash_result(file.path(), input_url).await?; - handler.save_regression(report, None).await?; + let crash_test_result = handler.get_crash_result(file.path(), input_url).await?; + RegressionReport { + crash_test_result, + original_crash_test_result: None, + } + .save(None, regression_reports) + .await? } Ok(()) } -/// Run the regression on the reports in the 'inputs_reports' location -/// * `heartbeat_client` - heartbeat client -/// * `input_reports` - location of the reports used in this regression run -/// * `report_list` - list of report file names selected to be used in the regression -/// * `crashes` - location of the crash files referenced by the reports in input_reports pub async fn handle_crash_reports( - heartbeat_client: &Option, - input_reports: &SyncedDir, - report_list: &[String], - crashes: &SyncedDir, handler: &impl RegressionHandler, + crashes: &SyncedDir, + report_dirs: &[&SyncedDir], + report_list: &Option>, + regression_reports: &SyncedDir, + heartbeat_client: &Option, ) -> Result<()> { - if report_list.is_empty() { - input_reports.sync_pull().await?; - } else { - for file in report_list { - let input_url = input_reports - .url - .clone() - .ok_or_else(|| format_err!("no input url"))? - .blob(file); - download_input(input_url.url(), &input_reports.path).await?; - } + // without crash report containers, skip this method + if report_dirs.is_empty() { + return Ok(()); } - let mut report_files = tokio::fs::read_dir(&input_reports.path).await?; - while let Some(file) = report_files.next_entry().await? { - heartbeat_client.alive(); - let crash_report_str = std::fs::read_to_string(file.path())?; - let crash_report: CrashReport = serde_json::from_str(&crash_report_str)?; - let input_url = crash_report.input_blob.clone().and_then(|input_blob| { - let crashes_url = crashes.url.clone()?; - Some(crashes_url.blob(input_blob.name).url()) - }); - if let Some(input_url) = input_url { - let input = download_input(input_url.clone(), &crashes.path).await?; - let report = handler.get_crash_result(input, Some(input_url)).await?; + crashes.init_pull().await?; + + for possible_dir in report_dirs { + possible_dir.init_pull().await?; - handler.save_regression(report, Some(crash_report)).await?; + let mut report_files = tokio::fs::read_dir(&possible_dir.path).await?; + while let Some(file) = report_files.next_entry().await? { + heartbeat_client.alive(); + let file_path = file.path(); + if !file_path.is_file() { + continue; + } + + let file_name = file_path + .file_name() + .ok_or_else(|| format_err!("missing filename"))? + .to_string_lossy() + .to_string(); + + if let Some(report_list) = &report_list { + if !report_list.contains(&file_name) { + continue; + } + } + + let original_crash_test_result = parse_report_file(file.path()) + .await + .with_context(|| format!("unable to parse crash report: {}", file_name))?; + + let input_blob = match &original_crash_test_result { + CrashTestResult::CrashReport(x) => x.input_blob.clone(), + CrashTestResult::NoRepro(x) => x.input_blob.clone(), + } + .ok_or_else(|| format_err!("crash report is missing input blob: {}", file_name))?; + + let input_url = crashes.url.clone().map(|x| x.blob(&input_blob.name).url()); + let input = crashes.path.join(&input_blob.name); + let crash_test_result = handler.get_crash_result(input, input_url).await?; + + RegressionReport { + crash_test_result, + original_crash_test_result: Some(original_crash_test_result), + } + .save(Some(file_name), regression_reports) + .await? } } + Ok(()) } diff --git a/src/agent/onefuzz-agent/src/tasks/regression/generic.rs b/src/agent/onefuzz-agent/src/tasks/regression/generic.rs index d48ed04c2b..e872618102 100644 --- a/src/agent/onefuzz-agent/src/tasks/regression/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/regression/generic.rs @@ -3,10 +3,7 @@ use crate::tasks::{ config::CommonConfig, - report::{ - crash_report::{CrashReport, CrashTestResult}, - generic, - }, + report::{crash_report::CrashTestResult, generic}, utils::default_bool_true, }; use anyhow::Result; @@ -28,18 +25,15 @@ pub struct Config { #[serde(default)] pub target_env: HashMap, - pub inputs: Option, - - pub input_reports: Option, - pub crashes: Option, - - #[serde(default)] - pub report_list: Vec, + pub target_timeout: Option, - pub no_repro: Option, + pub crashes: SyncedDir, + pub regression_reports: SyncedDir, + pub report_list: Option>, pub reports: Option, - - pub target_timeout: Option, + pub unique_reports: Option, + pub no_repro: Option, + pub readonly_inputs: Option, #[serde(default)] pub check_asan_log: bool, @@ -79,21 +73,6 @@ impl RegressionHandler for GenericRegressionTask { }; generic::test_input(args).await } - - async fn save_regression( - &self, - crash_result: CrashTestResult, - original_report: Option, - ) -> Result<()> { - crash_result - .save_regression( - original_report, - &self.config.reports, - &self.config.no_repro, - format!("{}/", self.config.common.task_id), - ) - .await - } } impl GenericRegressionTask { @@ -104,12 +83,24 @@ impl GenericRegressionTask { pub async fn run(&self) -> Result<()> { info!("Starting generic regression task"); let heartbeat_client = self.config.common.init_heartbeat().await?; + + let mut report_dirs = vec![]; + for dir in &[ + &self.config.reports, + &self.config.unique_reports, + &self.config.no_repro, + ] { + if let Some(dir) = dir { + report_dirs.push(dir); + } + } common::run( heartbeat_client, - &self.config.input_reports, - &self.config.report_list, + &self.config.regression_reports, &self.config.crashes, - &self.config.inputs, + &report_dirs, + &self.config.report_list, + &self.config.readonly_inputs, self, ) .await?; diff --git a/src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs b/src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs index e55657c4b9..ae53cd5f63 100644 --- a/src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs +++ b/src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs @@ -3,10 +3,7 @@ use crate::tasks::{ config::CommonConfig, - report::{ - crash_report::{CrashReport, CrashTestResult}, - libfuzzer_report, - }, + report::{crash_report::CrashTestResult, libfuzzer_report}, utils::default_bool_true, }; @@ -29,18 +26,15 @@ pub struct Config { #[serde(default)] pub target_env: HashMap, - pub inputs: Option, - - pub input_reports: Option, - pub crashes: Option, - - #[serde(default)] - pub report_list: Vec, + pub target_timeout: Option, - pub no_repro: Option, + pub crashes: SyncedDir, + pub regression_reports: SyncedDir, + pub report_list: Option>, + pub unique_reports: Option, pub reports: Option, - - pub target_timeout: Option, + pub no_repro: Option, + pub readonly_inputs: Option, #[serde(default = "default_bool_true")] pub check_fuzzer_help: bool, @@ -76,21 +70,6 @@ impl RegressionHandler for LibFuzzerRegressionTask { }; libfuzzer_report::test_input(args).await } - - async fn save_regression( - &self, - crash_result: CrashTestResult, - original_report: Option, - ) -> Result<()> { - crash_result - .save_regression( - original_report, - &self.config.reports, - &self.config.no_repro, - format!("{}/", self.config.common.task_id), - ) - .await - } } impl LibFuzzerRegressionTask { @@ -100,13 +79,26 @@ impl LibFuzzerRegressionTask { pub async fn run(&self) -> Result<()> { info!("Starting libfuzzer regression task"); + + let mut report_dirs = vec![]; + for dir in &[ + &self.config.reports, + &self.config.unique_reports, + &self.config.no_repro, + ] { + if let Some(dir) = dir { + report_dirs.push(dir); + } + } + let heartbeat_client = self.config.common.init_heartbeat().await?; common::run( heartbeat_client, - &self.config.input_reports, - &self.config.report_list, + &self.config.regression_reports, &self.config.crashes, - &self.config.inputs, + &report_dirs, + &self.config.report_list, + &self.config.readonly_inputs, self, ) .await?; diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index 4641a39462..ab43cc7d3a 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -65,6 +65,7 @@ pub struct NoCrash { } #[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] pub enum CrashTestResult { CrashReport(CrashReport), NoRepro(NoCrash), @@ -72,8 +73,32 @@ pub enum CrashTestResult { #[derive(Debug, Deserialize, Serialize)] pub struct RegressionReport { - crash_result: CrashTestResult, - original_report: Option, + pub crash_test_result: CrashTestResult, + pub original_crash_test_result: Option, +} + +impl RegressionReport { + pub async fn save( + self, + report_name: Option, + regression_reports: &SyncedDir, + ) -> Result<()> { + let (event, name) = match &self.crash_test_result { + CrashTestResult::CrashReport(report) => { + let name = report_name.unwrap_or_else(|| report.unique_blob_name()); + (regression_report, name) + } + CrashTestResult::NoRepro(report) => { + let name = report_name.unwrap_or_else(|| report.blob_name()); + (regression_unable_to_reproduce, name) + } + }; + + if upload_or_save_local(&self, &name, regression_reports).await? { + event!(event; EventData::Path = name); + } + Ok(()) + } } // Conditionally upload a report, if it would not be a duplicate. @@ -114,7 +139,7 @@ async fn upload_or_save_local( } impl CrashTestResult { - /// Saves teh crash result as a crash report + /// Saves the crash result as a crash report /// * `unique_reports` - location to save the deduplicated report if the bug was reproduced /// * `reports` - location to save the report if the bug was reproduced /// * `no_repro` - location to save the report if the bug was not reproduced @@ -153,50 +178,6 @@ impl CrashTestResult { } Ok(()) } - - /// Saves teh crash result as a regression report - /// * `original_report` - optional original crash report used in this regression - /// * `reports` - location to save the report if the bug was reproduced - /// * `no_repro` - location to save the report if the bug was not reproduced - /// * `prefix` - prefix of the report file name - pub async fn save_regression( - self, - original_report: Option, - reports: &Option, - no_repro: &Option, - prefix: impl AsRef, - ) -> Result<()> { - match self { - Self::CrashReport(report) => { - // Use SHA-256 of call stack as dedupe key. - if let Some(reports) = reports { - let name = format!("{}{}", prefix.as_ref(), report.unique_blob_name()); - let report = RegressionReport { - crash_result: Self::CrashReport(report), - original_report, - }; - - if upload_or_save_local(&report, &name, reports).await? { - event!(regression_report; EventData::Path = name); - } - } - } - - Self::NoRepro(report) => { - if let Some(no_repro) = no_repro { - let name = format!("{}{}", prefix.as_ref(), report.blob_name()); - let report = RegressionReport { - crash_result: Self::NoRepro(report), - original_report, - }; - if upload_or_save_local(&report, &name, no_repro).await? { - event!(regression_unable_to_reproduce; EventData::Path = name); - } - } - } - } - Ok(()) - } } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -256,7 +237,7 @@ impl NoCrash { } } -async fn parse_report_file(path: PathBuf) -> Result { +pub async fn parse_report_file(path: PathBuf) -> Result { let raw = std::fs::read_to_string(&path) .with_context(|| format_err!("unable to open crash report: {}", path.display()))?; diff --git a/src/api-service/__app__/onefuzzlib/notifications/ado.py b/src/api-service/__app__/onefuzzlib/notifications/ado.py index ee6b90c399..3d9e67f9df 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/ado.py +++ b/src/api-service/__app__/onefuzzlib/notifications/ado.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import logging -from typing import Iterator, List, Optional +from typing import Iterator, List, Optional, Union from azure.devops.connection import Connection from azure.devops.credentials import BasicAuthentication @@ -24,7 +24,7 @@ WorkItemTrackingClient, ) from memoization import cached -from onefuzztypes.models import ADOTemplate, Report +from onefuzztypes.models import ADOTemplate, RegressionReport, Report from onefuzztypes.primitives import Container from ..secrets import get_secret_string_value @@ -51,7 +51,11 @@ def get_valid_fields( class ADO: def __init__( - self, container: Container, filename: str, config: ADOTemplate, report: Report + self, + container: Container, + filename: str, + config: ADOTemplate, + report: Report, ): self.config = config self.renderer = Render(container, filename, report) @@ -203,8 +207,20 @@ def process(self) -> None: def notify_ado( - config: ADOTemplate, container: Container, filename: str, report: Report + config: ADOTemplate, + container: Container, + filename: str, + report: Union[Report, RegressionReport], ) -> None: + if isinstance(report, RegressionReport): + logging.info( + "ado integration does not support regression reports. " + "container:%s filename:%s", + container, + filename, + ) + return + logging.info( "notify ado: job_id:%s task_id:%s container:%s filename:%s", report.job_id, diff --git a/src/api-service/__app__/onefuzzlib/notifications/github_issues.py b/src/api-service/__app__/onefuzzlib/notifications/github_issues.py index de048b2e49..e17f94d6a6 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/github_issues.py +++ b/src/api-service/__app__/onefuzzlib/notifications/github_issues.py @@ -4,13 +4,18 @@ # Licensed under the MIT License. import logging -from typing import List, Optional +from typing import List, Optional, Union from github3 import login from github3.exceptions import GitHubException from github3.issues import Issue from onefuzztypes.enums import GithubIssueSearchMatch -from onefuzztypes.models import GithubAuth, GithubIssueTemplate, Report +from onefuzztypes.models import ( + GithubAuth, + GithubIssueTemplate, + RegressionReport, + Report, +) from onefuzztypes.primitives import Container from ..secrets import get_secret_obj @@ -107,10 +112,18 @@ def github_issue( config: GithubIssueTemplate, container: Container, filename: str, - report: Optional[Report], + report: Optional[Union[Report, RegressionReport]], ) -> None: if report is None: return + if isinstance(report, RegressionReport): + logging.info( + "github issue integration does not support regression reports. " + "container:%s filename:%s", + container, + filename, + ) + return try: handler = GithubIssue(config, container, filename, report) diff --git a/src/api-service/__app__/onefuzzlib/notifications/main.py b/src/api-service/__app__/onefuzzlib/notifications/main.py index 64c7e963a4..50a90be58d 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/main.py +++ b/src/api-service/__app__/onefuzzlib/notifications/main.py @@ -10,12 +10,18 @@ from memoization import cached from onefuzztypes import models from onefuzztypes.enums import ErrorCode, TaskState -from onefuzztypes.events import EventCrashReported, EventFileAdded +from onefuzztypes.events import ( + EventCrashReported, + EventFileAdded, + EventRegressionReported, +) from onefuzztypes.models import ( ADOTemplate, Error, GithubIssueTemplate, NotificationTemplate, + RegressionReport, + Report, Result, TeamsTemplate, ) @@ -26,7 +32,7 @@ from ..azure.storage import StorageType from ..events import send_event from ..orm import ORMMixin -from ..reports import get_report +from ..reports import get_report_or_regression from ..tasks.config import get_input_container_queues from ..tasks.main import Task from .ado import notify_ado @@ -102,7 +108,7 @@ def get_queue_tasks() -> Sequence[Tuple[Task, Sequence[str]]]: def new_files(container: Container, filename: str) -> None: - report = get_report(container, filename) + report = get_report_or_regression(container, filename) notifications = get_notifications(container) if notifications: @@ -134,9 +140,15 @@ def new_files(container: Container, filename: str) -> None: ) send_message(task.task_id, bytes(url, "utf-8"), StorageType.corpus) - if report: + if isinstance(report, Report): send_event( EventCrashReported(report=report, container=container, filename=filename) ) + elif isinstance(report, RegressionReport): + send_event( + EventRegressionReported( + regression_report=report, container=container, filename=filename + ) + ) else: send_event(EventFileAdded(container=container, filename=filename)) diff --git a/src/api-service/__app__/onefuzzlib/notifications/teams.py b/src/api-service/__app__/onefuzzlib/notifications/teams.py index 542f704ecf..f8ed068402 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/teams.py +++ b/src/api-service/__app__/onefuzzlib/notifications/teams.py @@ -4,10 +4,10 @@ # Licensed under the MIT License. import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union import requests -from onefuzztypes.models import Report, TeamsTemplate +from onefuzztypes.models import RegressionReport, Report, TeamsTemplate from onefuzztypes.primitives import Container from ..azure.containers import auth_download_url @@ -54,12 +54,15 @@ def send_teams_webhook( def notify_teams( - config: TeamsTemplate, container: Container, filename: str, report: Optional[Report] + config: TeamsTemplate, + container: Container, + filename: str, + report: Optional[Union[Report, RegressionReport]], ) -> None: text = None facts: List[Dict[str, str]] = [] - if report: + if isinstance(report, Report): task = Task.get(report.job_id, report.task_id) if not task: logging.error( diff --git a/src/api-service/__app__/onefuzzlib/reports.py b/src/api-service/__app__/onefuzzlib/reports.py index 9c628b0445..0a996ea8bd 100644 --- a/src/api-service/__app__/onefuzzlib/reports.py +++ b/src/api-service/__app__/onefuzzlib/reports.py @@ -8,7 +8,7 @@ from typing import Optional, Union from memoization import cached -from onefuzztypes.models import Report +from onefuzztypes.models import RegressionReport, Report from onefuzztypes.primitives import Container from pydantic import ValidationError @@ -16,17 +16,16 @@ from .azure.storage import StorageType -def parse_report( +def parse_report_or_regression( content: Union[str, bytes], file_path: Optional[str] = None -) -> Optional[Report]: +) -> Optional[Union[Report, RegressionReport]]: if isinstance(content, bytes): try: content = content.decode() except UnicodeDecodeError as err: logging.error( - "unable to parse report (%s): unicode decode of report failed - %s", - file_path, - err, + f"unable to parse report ({file_path}): " + f"unicode decode of report failed - {err}" ) return None @@ -34,22 +33,31 @@ def parse_report( data = json.loads(content) except json.decoder.JSONDecodeError as err: logging.error( - "unable to parse report (%s): json decoding failed - %s", file_path, err + f"unable to parse report ({file_path}): json decoding failed - {err}" ) return None + regression_err = None try: - entry = Report.parse_obj(data) + return RegressionReport.parse_obj(data) except ValidationError as err: - logging.error("unable to parse report (%s): %s", file_path, err) - return None + regression_err = err - return entry + try: + return Report.parse_obj(data) + except ValidationError as err: + logging.error( + f"unable to parse report ({file_path}) as a report or regression. " + f"regression error: {regression_err} report error: {err}" + ) + return None # cache the last 1000 reports @cached(max_size=1000) -def get_report(container: Container, filename: str) -> Optional[Report]: +def get_report_or_regression( + container: Container, filename: str +) -> Optional[Union[Report, RegressionReport]]: file_path = "/".join([container, filename]) if not filename.endswith(".json"): logging.error("get_report invalid extension: %s", file_path) @@ -60,4 +68,11 @@ def get_report(container: Container, filename: str) -> Optional[Report]: logging.error("get_report invalid blob: %s", file_path) return None - return parse_report(blob, file_path=file_path) + return parse_report_or_regression(blob, file_path=file_path) + + +def get_report(container: Container, filename: str) -> Optional[Report]: + result = get_report_or_regression(container, filename) + if isinstance(result, Report): + return result + return None diff --git a/src/api-service/__app__/onefuzzlib/tasks/defs.py b/src/api-service/__app__/onefuzzlib/tasks/defs.py index 40b3aa5845..619922e1ed 100644 --- a/src/api-service/__app__/onefuzzlib/tasks/defs.py +++ b/src/api-service/__app__/onefuzzlib/tasks/defs.py @@ -434,10 +434,14 @@ permissions=[ContainerPermission.Read, ContainerPermission.List], ), ContainerDefinition( - type=ContainerType.input_reports, + type=ContainerType.regression_reports, compare=Compare.Equal, value=1, - permissions=[ContainerPermission.Read, ContainerPermission.List], + permissions=[ + ContainerPermission.Write, + ContainerPermission.Read, + ContainerPermission.List, + ], ), ContainerDefinition( type=ContainerType.crashes, @@ -449,16 +453,22 @@ type=ContainerType.reports, compare=Compare.AtMost, value=1, - permissions=[ContainerPermission.Write], + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.unique_reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], ), ContainerDefinition( type=ContainerType.no_repro, compare=Compare.AtMost, value=1, - permissions=[ContainerPermission.Write], + permissions=[ContainerPermission.Read, ContainerPermission.List], ), ContainerDefinition( - type=ContainerType.inputs, + type=ContainerType.readonly_inputs, compare=Compare.AtMost, value=1, permissions=[ @@ -487,10 +497,14 @@ permissions=[ContainerPermission.Read, ContainerPermission.List], ), ContainerDefinition( - type=ContainerType.input_reports, + type=ContainerType.regression_reports, compare=Compare.Equal, value=1, - permissions=[ContainerPermission.Read, ContainerPermission.List], + permissions=[ + ContainerPermission.Write, + ContainerPermission.Read, + ContainerPermission.List, + ], ), ContainerDefinition( type=ContainerType.crashes, @@ -498,20 +512,26 @@ value=1, permissions=[ContainerPermission.Read, ContainerPermission.List], ), + ContainerDefinition( + type=ContainerType.unique_reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), ContainerDefinition( type=ContainerType.reports, compare=Compare.AtMost, value=1, - permissions=[ContainerPermission.Write], + permissions=[ContainerPermission.Read, ContainerPermission.List], ), ContainerDefinition( type=ContainerType.no_repro, compare=Compare.AtMost, value=1, - permissions=[ContainerPermission.Write], + permissions=[ContainerPermission.Read, ContainerPermission.List], ), ContainerDefinition( - type=ContainerType.inputs, + type=ContainerType.readonly_inputs, compare=Compare.AtMost, value=1, permissions=[ diff --git a/src/api-service/tests/test_report_parse.py b/src/api-service/tests/test_report_parse.py index c7d282c771..de9ce76f12 100755 --- a/src/api-service/tests/test_report_parse.py +++ b/src/api-service/tests/test_report_parse.py @@ -10,7 +10,7 @@ from onefuzztypes.models import Report -from __app__.onefuzzlib.reports import parse_report +from __app__.onefuzzlib.reports import parse_report_or_regression class TestReportParse(unittest.TestCase): @@ -20,16 +20,15 @@ def test_sample(self) -> None: data = json.load(handle) invalid = {"unused_field_1": 3} - report = parse_report(json.dumps(data)) + report = parse_report_or_regression(json.dumps(data)) self.assertIsInstance(report, Report) with self.assertLogs(level="ERROR"): - self.assertIsNone(parse_report('"invalid"')) + self.assertIsNone(parse_report_or_regression('"invalid"')) with self.assertLogs(level="WARNING") as logs: - self.assertIsNone(parse_report(json.dumps(invalid))) - - self.assertTrue(any(["unable to parse report" in x for x in logs.output])) + self.assertIsNone(parse_report_or_regression(json.dumps(invalid))) + self.assertTrue(any(["unable to parse report" in x for x in logs.output])) if __name__ == "__main__": diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index bc24ebd6de..886b56cacd 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -40,6 +40,12 @@ def _build_container_name( build=build, platform=platform.name, ) + elif container_type == ContainerType.regression_reports: + guid = onefuzz.utils.namespaced_guid( + project, + name, + build=build, + ) else: guid = onefuzz.utils.namespaced_guid(project, name) diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index 8b2485a351..63013c5cc0 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -61,6 +61,36 @@ def _create_tasks( expect_crash_on_failure: bool = True, ) -> None: + regression_containers = [ + (ContainerType.setup, containers[ContainerType.setup]), + (ContainerType.crashes, containers[ContainerType.crashes]), + (ContainerType.unique_reports, containers[ContainerType.unique_reports]), + ( + ContainerType.regression_reports, + containers[ContainerType.regression_reports], + ), + ] + + self.logger.info("creating libfuzzer_regression task") + regression_task = self.onefuzz.tasks.create( + job.job_id, + TaskType.libfuzzer_regression, + target_exe, + regression_containers, + pool_name=pool_name, + duration=duration, + vm_count=1, + reboot_after_setup=reboot_after_setup, + target_options=target_options, + target_env=target_env, + tags=tags, + target_timeout=crash_report_timeout, + check_retry_count=check_retry_count, + check_fuzzer_help=check_fuzzer_help, + debug=debug, + colocate=colocate_all_tasks or colocate_secondary_tasks, + ) + fuzzer_containers = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.crashes, containers[ContainerType.crashes]), @@ -92,7 +122,7 @@ def _create_tasks( expect_crash_on_failure=expect_crash_on_failure, ) - prereq_tasks = [fuzzer_task.task_id] + prereq_tasks = [fuzzer_task.task_id, regression_task.task_id] coverage_containers = [ (ContainerType.setup, containers[ContainerType.setup]), @@ -219,6 +249,7 @@ def basic( ContainerType.no_repro, ContainerType.coverage, ContainerType.unique_inputs, + ContainerType.regression_reports, ) if existing_inputs: diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index edc195c7bf..6a5c166efb 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -7,7 +7,7 @@ from onefuzztypes.enums import ContainerType, TaskDebugFlag, TaskType from onefuzztypes.models import Job, NotificationConfig -from onefuzztypes.primitives import Container, Directory, File, PoolName +from onefuzztypes.primitives import Directory, File, PoolName from onefuzz.api import Command @@ -29,9 +29,8 @@ def generic( build: str, pool_name: PoolName, *, - crashes: Container = None, - input_reports: Container = None, - inputs: Optional[Directory] = None, + reports: Optional[List[str]] = None, + crashes: Optional[List[File]] = None, target_exe: File = File("fuzz.exe"), tags: Optional[Dict[str, str]] = None, notification_config: Optional[NotificationConfig] = None, @@ -41,7 +40,6 @@ def generic( target_options: Optional[List[str]] = None, dryrun: bool = False, duration: int = 24, - report_list: Optional[List[str]] = None, crash_report_timeout: Optional[int] = None, debug: Optional[List[TaskDebugFlag]] = None, check_retry_count: Optional[int] = None, @@ -52,8 +50,8 @@ def generic( """ generic regression task - :param Container input_reports: Specify the container of the crash reports used in the regression - :param Container crashes: Specify the container of the input files of the crashes + :param File inputs: Specify a directory of inptus to use in the regression task + :param str reports: Specify specific report names to verify in the regression task :param bool fail_on_repro: Specify wether or not to throw an exception if a repro was generated :param bool delete_input_container: Specify wether or not to delete the input container """ @@ -64,25 +62,23 @@ def generic( name, build, pool_name, - crashes, - input_reports, - inputs, - target_exe, - tags, - notification_config, - target_env, - setup_dir, - reboot_after_setup, - target_options, - dryrun, - duration, - report_list, - crash_report_timeout, - debug, - check_retry_count, - check_fuzzer_help, - fail_on_repro, - delete_input_container, + crashes=crashes, + reports=reports, + target_exe=target_exe, + tags=tags, + notification_config=notification_config, + target_env=target_env, + setup_dir=setup_dir, + reboot_after_setup=reboot_after_setup, + target_options=target_options, + dryrun=dryrun, + duration=duration, + crash_report_timeout=crash_report_timeout, + debug=debug, + check_retry_count=check_retry_count, + check_fuzzer_help=check_fuzzer_help, + fail_on_repro=fail_on_repro, + delete_input_container=delete_input_container, ) def libfuzzer( @@ -92,9 +88,8 @@ def libfuzzer( build: str, pool_name: PoolName, *, - crashes: Container = None, - input_reports: Container = None, - inputs: Optional[Directory] = None, + reports: Optional[List[str]] = None, + crashes: Optional[List[File]] = None, target_exe: File = File("fuzz.exe"), tags: Optional[Dict[str, str]] = None, notification_config: Optional[NotificationConfig] = None, @@ -104,7 +99,6 @@ def libfuzzer( target_options: Optional[List[str]] = None, dryrun: bool = False, duration: int = 24, - report_list: Optional[List[str]] = None, crash_report_timeout: Optional[int] = None, debug: Optional[List[TaskDebugFlag]] = None, check_retry_count: Optional[int] = None, @@ -116,8 +110,8 @@ def libfuzzer( """ generic regression task - :param Container input_reports: Specify the container of the crash reports used in the regression - :param Container crashes: Specify the container of the input files of the crashes + :param File inputs: Specify a directory of inptus to use in the regression task + :param str reports: Specify specific report names to verify in the regression task :param bool fail_on_repro: Specify wether or not to throw an exception if a repro was generated :param bool delete_input_container: Specify wether or not to delete the input container """ @@ -128,25 +122,23 @@ def libfuzzer( name, build, pool_name, - crashes, - input_reports, - inputs, - target_exe, - tags, - notification_config, - target_env, - setup_dir, - reboot_after_setup, - target_options, - dryrun, - duration, - report_list, - crash_report_timeout, - debug, - check_retry_count, - check_fuzzer_help, - fail_on_repro, - delete_input_container, + crashes=crashes, + reports=reports, + target_exe=target_exe, + tags=tags, + notification_config=notification_config, + target_env=target_env, + setup_dir=setup_dir, + reboot_after_setup=reboot_after_setup, + target_options=target_options, + dryrun=dryrun, + duration=duration, + crash_report_timeout=crash_report_timeout, + debug=debug, + check_retry_count=check_retry_count, + check_fuzzer_help=check_fuzzer_help, + fail_on_repro=fail_on_repro, + delete_input_container=delete_input_container, ) def _create_job( @@ -156,9 +148,9 @@ def _create_job( name: str, build: str, pool_name: PoolName, - crashes: Container = None, - input_reports: Container = None, - inputs: Optional[Directory] = None, + *, + crashes: Optional[List[File]] = None, + reports: Optional[List[str]] = None, target_exe: File = File("fuzz.exe"), tags: Optional[Dict[str, str]] = None, notification_config: Optional[NotificationConfig] = None, @@ -168,7 +160,6 @@ def _create_job( target_options: Optional[List[str]] = None, dryrun: bool = False, duration: int = 24, - report_list: Optional[List[str]] = None, crash_report_timeout: Optional[int] = None, debug: Optional[List[TaskDebugFlag]] = None, check_retry_count: Optional[int] = None, @@ -177,11 +168,6 @@ def _create_job( delete_input_container: bool = True, ) -> Optional[Job]: - if not ((crashes and input_reports) or inputs): - self.logger.error( - "please specify either the 'crash' and 'input_reports' parameters or the 'inputs' parameter" - ) - if dryrun: return None @@ -198,40 +184,44 @@ def _create_job( target_exe=target_exe, ) - if inputs: - helper.containers[ - ContainerType.readonly_inputs - ] = helper.get_unique_container_name(ContainerType.readonly_inputs) - helper.define_containers( ContainerType.setup, + ContainerType.crashes, ContainerType.reports, ContainerType.no_repro, + ContainerType.unique_reports, + ContainerType.regression_reports, ) + if crashes: + helper.containers[ + ContainerType.readonly_inputs + ] = helper.get_unique_container_name(ContainerType.readonly_inputs) + helper.create_containers() + if crashes: + for file in crashes: + self.onefuzz.containers.files.upload_file( + helper.containers[ContainerType.unique_inputs], file + ) + helper.setup_notifications(notification_config) - if inputs: - helper.upload_inputs(inputs, read_only=True) containers = [ (ContainerType.setup, helper.containers[ContainerType.setup]), + (ContainerType.crashes, helper.containers[ContainerType.crashes]), (ContainerType.reports, helper.containers[ContainerType.reports]), (ContainerType.no_repro, helper.containers[ContainerType.no_repro]), + ( + ContainerType.unique_reports, + helper.containers[ContainerType.unique_reports], + ), + ( + ContainerType.regression_reports, + helper.containers[ContainerType.regression_reports], + ), ] - if crashes: - if self.onefuzz.containers.get(crashes): - containers.append((ContainerType.crashes, crashes)) - else: - self.logger.error(f"invalid crash container {crashes}") - - if input_reports: - if self.onefuzz.containers.get("input_reports"): - containers.append((ContainerType.input_reports, input_reports)) - else: - self.logger.error(f"invalid crash container {input_reports}") - helper.upload_setup(setup_dir, target_exe) target_exe_blob_name = helper.target_exe_blob_name(target_exe, setup_dir) @@ -252,7 +242,7 @@ def _create_job( check_retry_count=check_retry_count, debug=debug, check_fuzzer_help=check_fuzzer_help, - report_list=report_list, + report_list=reports, ) helper.wait_for_stopping = fail_on_repro @@ -267,7 +257,7 @@ def _create_job( repro_count = len( self.onefuzz.containers.files.list( - helper.containers[ContainerType.reports], + helper.containers[ContainerType.regression_reports], prefix=str(regression_task.task_id), ).files ) diff --git a/src/integration-tests/libfuzzer-regression/Makefile b/src/integration-tests/libfuzzer-regression/Makefile new file mode 100644 index 0000000000..b8a0f6e842 --- /dev/null +++ b/src/integration-tests/libfuzzer-regression/Makefile @@ -0,0 +1,16 @@ +CC=clang + +CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address + +all: broken.exe fixed.exe + +broken.exe: simple.c + $(CC) $(CFLAGS) simple.c -o broken.exe + +fixed.exe: simple.c + $(CC) $(CFLAGS) simple.c -o fixed.exe -DFIXED + +.PHONY: clean + +clean: + rm -f broken.exe fixed.exe diff --git a/src/integration-tests/libfuzzer-regression/seeds/good.txt b/src/integration-tests/libfuzzer-regression/seeds/good.txt new file mode 100644 index 0000000000..12799ccbe7 --- /dev/null +++ b/src/integration-tests/libfuzzer-regression/seeds/good.txt @@ -0,0 +1 @@ +good diff --git a/src/integration-tests/libfuzzer-regression/simple.c b/src/integration-tests/libfuzzer-regression/simple.c new file mode 100644 index 0000000000..8b02c742df --- /dev/null +++ b/src/integration-tests/libfuzzer-regression/simple.c @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) { + int cnt = 0; + + if (len < 3) { + return 0; + } + + if (data[0] == 'x') { cnt++; } + if (data[1] == 'y') { cnt++; } + if (data[2] == 'z') { cnt++; } + +#ifndef FIXED + if (cnt >= 3) { + abort(); + } +#endif + + return 0; +} diff --git a/src/pytypes/extra/generate-docs.py b/src/pytypes/extra/generate-docs.py index f551812be2..c2c39c4468 100755 --- a/src/pytypes/extra/generate-docs.py +++ b/src/pytypes/extra/generate-docs.py @@ -31,6 +31,7 @@ EventProxyCreated, EventProxyDeleted, EventProxyFailed, + EventRegressionReported, EventScalesetCreated, EventScalesetDeleted, EventScalesetFailed, @@ -45,8 +46,10 @@ ) from onefuzztypes.models import ( BlobRef, + CrashTestResult, Error, JobConfig, + RegressionReport, Report, TaskConfig, TaskContainers, @@ -87,6 +90,24 @@ def main() -> None: ], tags={}, ) + report = Report( + input_blob=BlobRef( + account="contoso-storage-account", + container=Container("crashes"), + name="input.txt", + ), + executable="fuzz.exe", + crash_type="example crash report type", + crash_site="example crash site", + call_stack=["#0 line", "#1 line", "#2 line"], + call_stack_sha256=ZERO_SHA256, + input_sha256=EMPTY_SHA256, + asan_log="example asan log", + task_id=UUID(int=0), + job_id=UUID(int=0), + scariness_score=10, + scariness_description="example-scariness", + ) examples: List[Event] = [ EventPing(ping_id=UUID(int=0)), EventTaskCreated( @@ -193,27 +214,18 @@ def main() -> None: pool_name=PoolName("example"), state=NodeState.setting_up, ), + EventRegressionReported( + regression_report=RegressionReport( + crash_test_result=CrashTestResult(crash_report=report), + original_crash_test_result=CrashTestResult(crash_report=report), + ), + container=Container("container-name"), + filename="example.json", + ), EventCrashReported( container=Container("container-name"), filename="example.json", - report=Report( - input_blob=BlobRef( - account="contoso-storage-account", - container=Container("crashes"), - name="input.txt", - ), - executable="fuzz.exe", - crash_type="example crash report type", - crash_site="example crash site", - call_stack=["#0 line", "#1 line", "#2 line"], - call_stack_sha256=ZERO_SHA256, - input_sha256=EMPTY_SHA256, - asan_log="example asan log", - task_id=UUID(int=0), - job_id=UUID(int=0), - scariness_score=10, - scariness_description="example-scariness", - ), + report=report, ), EventFileAdded(container=Container("container-name"), filename="example.txt"), EventNodeHeartbeat(machine_id=UUID(int=0), pool_name=PoolName("example")), diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index f02c4fc72e..bb856ee851 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -210,7 +210,7 @@ class ContainerType(Enum): tools = "tools" unique_inputs = "unique_inputs" unique_reports = "unique_reports" - input_reports = "input_reports" + regression_reports = "regression_reports" @classmethod def reset_defaults(cls) -> List["ContainerType"]: @@ -223,8 +223,9 @@ def reset_defaults(cls) -> List["ContainerType"]: cls.readonly_inputs, cls.reports, cls.setup, - cls.unique_reports, cls.unique_inputs, + cls.unique_reports, + cls.regression_reports, ] @classmethod diff --git a/src/pytypes/onefuzztypes/events.py b/src/pytypes/onefuzztypes/events.py index fc78334a3f..9e43e3296c 100644 --- a/src/pytypes/onefuzztypes/events.py +++ b/src/pytypes/onefuzztypes/events.py @@ -11,7 +11,15 @@ from pydantic import BaseModel, Extra, Field from .enums import OS, Architecture, NodeState, TaskState, TaskType -from .models import AutoScaleConfig, Error, JobConfig, Report, TaskConfig, UserInfo +from .models import ( + AutoScaleConfig, + Error, + JobConfig, + RegressionReport, + Report, + TaskConfig, + UserInfo, +) from .primitives import Container, PoolName, Region from .responses import BaseResponse @@ -156,6 +164,12 @@ class EventCrashReported(BaseEvent): filename: str +class EventRegressionReported(BaseEvent): + regression_report: RegressionReport + container: Container + filename: str + + class EventFileAdded(BaseEvent): container: Container filename: str @@ -183,6 +197,7 @@ class EventFileAdded(BaseEvent): EventTaskStopped, EventTaskHeartbeat, EventCrashReported, + EventRegressionReported, EventFileAdded, ] @@ -207,6 +222,7 @@ class EventType(Enum): task_state_updated = "task_state_updated" task_stopped = "task_stopped" crash_reported = "crash_reported" + regression_reported = "regression_reported" file_added = "file_added" task_heartbeat = "task_heartbeat" node_heartbeat = "node_heartbeat" @@ -234,6 +250,7 @@ class EventType(Enum): EventType.task_heartbeat: EventTaskHeartbeat, EventType.task_stopped: EventTaskStopped, EventType.crash_reported: EventCrashReported, + EventType.regression_reported: EventRegressionReported, EventType.file_added: EventFileAdded, } diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 6bdc4f2cd6..e0ed35fb3f 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -252,6 +252,26 @@ class Report(BaseModel): scariness_description: Optional[str] +class NoReproReport(BaseModel): + input_sha256: str + input_blob: Optional[BlobRef] + executable: str + task_id: UUID + job_id: UUID + tries: int + error: Optional[str] + + +class CrashTestResult(BaseModel): + crash_report: Optional[Report] + no_repro: Optional[NoReproReport] + + +class RegressionReport(BaseModel): + crash_test_result: CrashTestResult + original_crash_test_result: Optional[CrashTestResult] + + class ADODuplicateTemplate(BaseModel): increment: List[str] comment: Optional[str] @@ -392,7 +412,7 @@ class TaskUnitConfig(BaseModel): tools: CONTAINER_DEF unique_inputs: CONTAINER_DEF unique_reports: CONTAINER_DEF - input_reports: CONTAINER_DEF + regression_reports: CONTAINER_DEF class Forward(BaseModel): From f6a426cc0782af6ba12f04d80815ca0d843ac923 Mon Sep 17 00:00:00 2001 From: bmc-msft <41130664+bmc-msft@users.noreply.github.com> Date: Wed, 10 Mar 2021 17:03:15 -0500 Subject: [PATCH 02/13] enable long-running integration tests (#654) --- src/integration-tests/integration-test.py | 722 ++++++++++++---------- 1 file changed, 403 insertions(+), 319 deletions(-) diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index f5f5db1d98..de945e8900 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -7,13 +7,13 @@ """ Launch multiple templates using samples to verify Onefuzz works end-to-end """ # NOTE: -# 1. This script uses pre-built fuzzing samples from the onefuzz-samples project. -# https://github.com/microsoft/onefuzz-samples/releases/latest +# 1. This script uses an unpacked version of the `integration-test-results` +# from the CI pipeline. # -# 2. This script will create new pools & managed scalesets during the testing by -# default. To use pre-existing pools, specify `--user_pools os=pool_name` +# Check out https://github.com/microsoft/onefuzz/actions/workflows/ +# ci.yml?query=branch%3Amain+is%3Asuccess # -# 3. For each stage, this script launches everything for the stage in batch, then +# 2. For each stage, this script launches everything for the stage in batch, then # checks on each of the created items for the stage. This batch processing # allows testing multiple components concurrently. @@ -30,7 +30,7 @@ from onefuzz.backend import ContainerWrapper, wait from onefuzz.cli import execute_api from onefuzztypes.enums import OS, ContainerType, TaskState, VmState -from onefuzztypes.models import Job, Pool, Repro, Scaleset +from onefuzztypes.models import Job, Pool, Repro, Scaleset, Task from onefuzztypes.primitives import Container, Directory, File, PoolName, Region from pydantic import BaseModel, Field @@ -39,6 +39,13 @@ BUILD = "0" +class TaskTestState(Enum): + not_running = "not_running" + running = "running" + stopped = "stopped" + failed = "failed" + + class TemplateType(Enum): libfuzzer = "libfuzzer" libfuzzer_dotnet = "libfuzzer_dotnet" @@ -54,7 +61,7 @@ class Integration(BaseModel): inputs: Optional[str] use_setup: bool = Field(default=False) nested_setup_dir: Optional[str] - wait_for_files: List[ContainerType] + wait_for_files: Dict[ContainerType, int] check_asan_log: Optional[bool] = Field(default=False) disable_check_debugger: Optional[bool] = Field(default=False) reboot_after_setup: Optional[bool] = Field(default=False) @@ -67,14 +74,18 @@ class Integration(BaseModel): os=OS.linux, target_exe="fuzz.exe", inputs="seeds", - wait_for_files=[ContainerType.unique_reports], + wait_for_files={ContainerType.unique_reports: 1}, ), "linux-libfuzzer": Integration( template=TemplateType.libfuzzer, os=OS.linux, target_exe="fuzz.exe", inputs="seeds", - wait_for_files=[ContainerType.unique_reports, ContainerType.coverage], + wait_for_files={ + ContainerType.unique_reports: 1, + ContainerType.coverage: 1, + ContainerType.inputs: 2, + }, reboot_after_setup=True, ), "linux-libfuzzer-dotnet": Integration( @@ -84,7 +95,8 @@ class Integration(BaseModel): nested_setup_dir="my-fuzzer", inputs="inputs", use_setup=True, - wait_for_files=[ContainerType.inputs, ContainerType.crashes], + wait_for_files={ContainerType.inputs: 2, ContainerType.crashes: 1}, + test_repro=False, ), "linux-libfuzzer-aarch64-crosscompile": Integration( template=TemplateType.libfuzzer_qemu_user, @@ -92,28 +104,28 @@ class Integration(BaseModel): target_exe="fuzz.exe", inputs="inputs", use_setup=True, - wait_for_files=[ContainerType.inputs, ContainerType.crashes], + wait_for_files={ContainerType.inputs: 2, ContainerType.crashes: 1}, test_repro=False, ), "linux-libfuzzer-rust": Integration( template=TemplateType.libfuzzer, os=OS.linux, target_exe="fuzz_target_1", - wait_for_files=[ContainerType.unique_reports, ContainerType.coverage], + wait_for_files={ContainerType.unique_reports: 1, ContainerType.coverage: 1}, ), "linux-trivial-crash": Integration( template=TemplateType.radamsa, os=OS.linux, target_exe="fuzz.exe", inputs="seeds", - wait_for_files=[ContainerType.unique_reports], + wait_for_files={ContainerType.unique_reports: 1}, ), "linux-trivial-crash-asan": Integration( template=TemplateType.radamsa, os=OS.linux, target_exe="fuzz.exe", inputs="seeds", - wait_for_files=[ContainerType.unique_reports], + wait_for_files={ContainerType.unique_reports: 1}, check_asan_log=True, disable_check_debugger=True, ), @@ -122,89 +134,53 @@ class Integration(BaseModel): os=OS.windows, target_exe="fuzz.exe", inputs="seeds", - wait_for_files=[ - ContainerType.unique_reports, - ContainerType.coverage, - ], + wait_for_files={ + ContainerType.inputs: 2, + ContainerType.unique_reports: 1, + ContainerType.coverage: 1, + }, ), "windows-trivial-crash": Integration( template=TemplateType.radamsa, os=OS.windows, target_exe="fuzz.exe", inputs="seeds", - wait_for_files=[ContainerType.unique_reports], + wait_for_files={ContainerType.unique_reports: 1}, ), } class TestOnefuzz: - def __init__( - self, - onefuzz: Onefuzz, - logger: logging.Logger, - *, - pool_size: int, - os_list: List[OS], - targets: List[str], - skip_cleanup: bool, - ) -> None: + def __init__(self, onefuzz: Onefuzz, logger: logging.Logger, test_id: UUID) -> None: self.of = onefuzz self.logger = logger self.pools: Dict[OS, Pool] = {} - self.project = "test-" + str(uuid4()).split("-")[0] - self.pool_size = pool_size - self.os = os_list - self.targets = targets - self.skip_cleanup = skip_cleanup - - # job_id -> Job - self.jobs: Dict[UUID, Job] = {} - - # job_id -> List[container_url] - self.containers: Dict[UUID, List[ContainerWrapper]] = {} - - # task_id -> job_id - self.tasks: Dict[UUID, UUID] = {} - - self.job_os: Dict[UUID, OS] = {} - - self.successful_jobs: Set[UUID] = set() - self.failed_jobs: Set[UUID] = set() - self.failed_repro: Set[UUID] = set() - - # job_id -> Repro - self.repros: Dict[UUID, Repro] = {} - - # job_id -> target - self.target_jobs: Dict[UUID, str] = {} + self.test_id = test_id + self.project = f"test-{self.test_id}" def setup( self, *, region: Optional[Region] = None, - user_pools: Optional[Dict[str, str]] = None, + pool_size: int, + os_list: List[OS], + ) -> None: + for entry in os_list: + name = PoolName(f"testpool-{entry.name}-{self.test_id}") + self.logger.info("creating pool: %s:%s", entry.name, name) + self.pools[entry] = self.of.pools.create(name, entry) + self.logger.info("creating scaleset for pool: %s", name) + self.of.scalesets.create(name, pool_size, region=region) + + def launch( + self, path: Directory, *, os_list: List[OS], targets: List[str], duration=int ) -> None: - for entry in self.os: - if user_pools and entry.name in user_pools: - self.logger.info( - "using existing pool: %s:%s", entry.name, user_pools[entry.name] - ) - self.pools[entry] = self.of.pools.get(user_pools[entry.name]) - else: - name = PoolName("pool-%s-%s" % (self.project, entry.name)) - self.logger.info("creating pool: %s:%s", entry.name, name) - self.pools[entry] = self.of.pools.create(name, entry) - self.logger.info("creating scaleset for pool: %s", name) - self.of.scalesets.create(name, self.pool_size, region=region) - - def launch(self, path: str) -> None: """ Launch all of the fuzzing templates """ - for target, config in TARGETS.items(): - if target not in self.targets: + if target not in targets: continue - if config.os not in self.os: + if config.os not in os_list: continue self.logger.info("launching: %s", target) @@ -230,7 +206,7 @@ def launch(self, path: str) -> None: target_exe=target_exe, inputs=inputs, setup_dir=setup, - duration=1, + duration=duration, vm_count=1, reboot_after_setup=config.reboot_after_setup or False, ) @@ -245,7 +221,7 @@ def launch(self, path: str) -> None: target_harness=config.target_exe, inputs=inputs, setup_dir=setup, - duration=1, + duration=duration, vm_count=1, ) elif config.template == TemplateType.libfuzzer_qemu_user: @@ -256,7 +232,7 @@ def launch(self, path: str) -> None: self.pools[config.os].name, inputs=inputs, target_exe=target_exe, - duration=1, + duration=duration, vm_count=1, ) elif config.template == TemplateType.radamsa: @@ -270,7 +246,7 @@ def launch(self, path: str) -> None: setup_dir=setup, check_asan_log=config.check_asan_log or False, disable_check_debugger=config.disable_check_debugger or False, - duration=1, + duration=duration, vm_count=1, ) elif config.template == TemplateType.afl: @@ -282,7 +258,7 @@ def launch(self, path: str) -> None: target_exe=target_exe, inputs=inputs, setup_dir=setup, - duration=1, + duration=duration, vm_count=1, ) else: @@ -291,21 +267,9 @@ def launch(self, path: str) -> None: if not job: raise Exception("missing job") - self.containers[job.job_id] = [] - for task in self.of.tasks.list(job_id=job.job_id): - self.tasks[task.task_id] = job.job_id - self.containers[job.job_id] += [ - ContainerWrapper(self.of.containers.get(x.name).sas_url) - for x in task.config.containers - if x.type in TARGETS[job.config.name].wait_for_files - ] - self.jobs[job.job_id] = job - self.job_os[job.job_id] = config.os - self.target_jobs[job.job_id] = target - - def check_task(self, task_id: UUID, scalesets: List[Scaleset]) -> Optional[str]: - task = self.of.tasks.get(task_id) - + def check_task( + self, job: Job, task: Task, scalesets: List[Scaleset] + ) -> TaskTestState: # Check if the scaleset the task is assigned is OK for scaleset in scalesets: if ( @@ -313,279 +277,335 @@ def check_task(self, task_id: UUID, scalesets: List[Scaleset]) -> Optional[str]: and scaleset.pool_name == task.config.pool.pool_name and scaleset.state not in scaleset.state.available() ): - return "task scaleset failed: %s - %s - %s (%s)" % ( - self.jobs[self.tasks[task_id]].config.name, + self.logger.error( + "task scaleset failed: %s - %s - %s (%s)", + job.config.name, task.config.task.type.name, scaleset.state.name, scaleset.error, ) + return TaskTestState.failed + + task = self.of.tasks.get(task.task_id) # check if the task itself has an error if task.error is not None: - return "task failed: %s - %s - %s (%s)" % ( - task_id, - self.jobs[self.tasks[task_id]].config.name, + self.logger.error( + "task failed: %s - %s (%s)", + job.config.name, task.config.task.type.name, task.error, ) + return TaskTestState.failed - # just in case someone else stopped the task - if task.state in TaskState.shutting_down(): - return "task shutdown early: %s - %s" % ( - self.jobs[self.tasks[task_id]].config.name, - task.config.task.type.name, - ) - return None + if task.state in [TaskState.stopped, TaskState.stopping]: + return TaskTestState.stopped + + if task.state == TaskState.running: + return TaskTestState.running + + return TaskTestState.not_running + + def check_jobs( + self, poll: bool = False, stop_on_complete_check: bool = False + ) -> bool: + """ Check all of the integration jobs """ + jobs: Dict[UUID, Job] = {x.job_id: x for x in self.get_jobs()} + job_tasks: Dict[UUID, List[Task]] = {} + check_containers: Dict[UUID, Dict[Container, Tuple[ContainerWrapper, int]]] = {} + + for job in jobs.values(): + if job.config.name not in TARGETS: + self.logger.error("unknown job target: %s", job.config.name) + continue + + tasks = self.of.jobs.tasks.list(job.job_id) + job_tasks[job.job_id] = tasks + check_containers[job.job_id] = {} + for task in tasks: + for container in task.config.containers: + if container.type in TARGETS[job.config.name].wait_for_files: + count = TARGETS[job.config.name].wait_for_files[container.type] + check_containers[job.job_id][container.name] = ( + ContainerWrapper( + self.of.containers.get(container.name).sas_url + ), + count, + ) + + self.success = True + self.logger.info("checking %d jobs", len(jobs)) - def check_jobs_impl( - self, - ) -> Tuple[bool, str, bool]: self.cleared = False def clear() -> None: if not self.cleared: self.cleared = True - print("") + if poll: + print("") + + def check_jobs_impl() -> Tuple[bool, str, bool]: + self.cleared = False + failed_jobs: Set[UUID] = set() + job_task_states: Dict[UUID, Set[TaskTestState]] = {} + + for job_id in check_containers: + finished_containers: Set[Container] = set() + for (container_name, container_impl) in check_containers[ + job_id + ].items(): + container_client, count = container_impl + if len(container_client.list_blobs()) >= count: + clear() + self.logger.info( + "found files for %s - %s", + jobs[job_id].config.name, + container_name, + ) + finished_containers.add(container_name) + + for container_name in finished_containers: + del check_containers[job_id][container_name] + + scalesets = self.of.scalesets.list() + for job_id in job_tasks: + finished_tasks: Set[UUID] = set() + job_task_states[job_id] = set() + + for task in job_tasks[job_id]: + if job_id not in jobs: + continue + + task_result = self.check_task(jobs[job_id], task, scalesets) + if task_result == TaskTestState.failed: + self.success = False + failed_jobs.add(job_id) + elif task_result == TaskTestState.stopped: + finished_tasks.add(task.task_id) + else: + job_task_states[job_id].add(task_result) + job_tasks[job_id] = [ + x for x in job_tasks[job_id] if x.task_id not in finished_tasks + ] - if self.jobs: - finished_job: Set[UUID] = set() + to_remove: Set[UUID] = set() + for job in jobs.values(): + # stop tracking failed jobs + if job.job_id in failed_jobs: + if job.job_id in check_containers: + del check_containers[job.job_id] + if job.job_id in job_tasks: + del job_tasks[job.job_id] + continue - # check all the containers we care about for the job - for job_id in self.containers: - done: Set[ContainerWrapper] = set() - for container in self.containers[job_id]: - if len(container.list_blobs()) > 0: + # stop checking containers once all the containers for the job + # have checked out. + if job.job_id in check_containers: + if not check_containers[job.job_id]: clear() self.logger.info( - "new files in: %s", container.client.container_name + "found files in all containers for %s", job.config.name ) - done.add(container) - for container in done: - self.containers[job_id].remove(container) - if not self.containers[job_id]: - clear() - self.logger.info("finished: %s", self.jobs[job_id].config.name) - finished_job.add(job_id) - - # check all the tasks associated with the job - if self.tasks: - scalesets = self.of.scalesets.list() - for task_id in self.tasks: - error = self.check_task(task_id, scalesets) - if error is not None: - clear() - self.logger.error(error) - finished_job.add(self.tasks[task_id]) - self.failed_jobs.add(self.tasks[task_id]) - - # cleanup jobs that are done testing - for job_id in finished_job: - self.stop_template( - self.jobs[job_id].config.name, delete_containers=False - ) + del check_containers[job.job_id] - for task_id, task_job_id in list(self.tasks.items()): - if job_id == task_job_id: - del self.tasks[task_id] + if job.job_id not in check_containers: + if job.job_id in job_task_states: + if set([TaskTestState.running]).issuperset( + job_task_states[job.job_id] + ): + del job_tasks[job.job_id] - if job_id in self.jobs: - self.successful_jobs.add(job_id) - del self.jobs[job_id] + if job.job_id not in job_tasks and job.job_id not in check_containers: + clear() + self.logger.info("%s completed", job.config.name) + to_remove.add(job.job_id) - if job_id in self.containers: - del self.containers[job_id] + for job_id in to_remove: + if stop_on_complete_check: + self.stop_job(jobs[job_id]) + del jobs[job_id] - msg = "waiting on: %s" % ",".join( - sorted(x.config.name for x in self.jobs.values()) - ) - if len(msg) > 80: - msg = "waiting on %d jobs" % len(self.jobs) + msg = "waiting on: %s" % ",".join( + sorted(x.config.name for x in jobs.values()) + ) + if poll and len(msg) > 80: + msg = "waiting on %d jobs" % len(jobs) - return ( - not bool(self.jobs), - msg, - not bool(self.failed_jobs), - ) + if not jobs: + msg = "done all tasks" - def check_jobs(self) -> bool: - """ Check all of the integration jobs """ - self.logger.info("checking jobs") - return wait(self.check_jobs_impl) + return (not bool(jobs), msg, self.success) - def get_job_crash(self, job_id: UUID) -> Optional[Tuple[Container, str]]: - # get the crash container for a given job + if poll: + return wait(check_jobs_impl) + else: + _, msg, result = check_jobs_impl() + self.logger.info(msg) + return result + def get_job_crash_report(self, job_id: UUID) -> Optional[Tuple[Container, str]]: for task in self.of.tasks.list(job_id=job_id, state=None): for container in task.config.containers: - if container.type != ContainerType.unique_reports: + if container.type not in [ + ContainerType.unique_reports, + ContainerType.reports, + ]: continue + files = self.of.containers.files.list(container.name) if len(files.files) > 0: return (container.name, files.files[0]) return None - def launch_repro(self) -> None: + def launch_repro(self) -> Tuple[bool, Dict[UUID, Tuple[Job, Repro]]]: # launch repro for one report from all succeessful jobs has_cdb = bool(which("cdb.exe")) has_gdb = bool(which("gdb")) - for job_id in self.successful_jobs: - if not TARGETS[self.target_jobs[job_id]].test_repro: - self.logger.info("skipping repro for %s", self.target_jobs[job_id]) + + jobs = self.get_jobs() + + result = True + repros = {} + for job in jobs: + if not TARGETS[job.config.name].test_repro: + self.logger.info("not testing repro for %s", job.config.name) continue - if self.job_os[job_id] == OS.linux and not has_gdb: + if TARGETS[job.config.name].os == OS.linux and not has_gdb: self.logger.warning( - "missing gdb in path, not launching repro: %s", - self.target_jobs[job_id], + "skipping repro for %s, missing gdb", job.config.name ) continue - if self.job_os[job_id] == OS.windows and not has_cdb: + if TARGETS[job.config.name].os == OS.windows and not has_cdb: self.logger.warning( - "missing cdb in path, not launching repro: %s", - self.target_jobs[job_id], + "skipping repro for %s, missing cdb", job.config.name ) continue - self.logger.info("launching repro: %s", self.target_jobs[job_id]) - report = self.get_job_crash(job_id) + report = self.get_job_crash_report(job.job_id) if report is None: - self.logger.warning( - "target does not include crash reports: %s", - self.target_jobs[job_id], + self.logger.error( + "target does not include crash reports: %s", job.config.name ) - return - (container, path) = report - self.repros[job_id] = self.of.repro.create(container, path, duration=1) + result = False + else: + self.logger.info("launching repro: %s", job.config.name) + (container, path) = report + repro = self.of.repro.create(container, path, duration=1) + repros[job.job_id] = (job, repro) - def check_repro_impl( - self, - ) -> Tuple[bool, str, bool]: - # check all of the launched repros + return (result, repros) - self.cleared = False + def check_repro(self, repros: Dict[UUID, Tuple[Job, Repro]]) -> bool: + self.logger.info("checking repros") + self.success = True - def clear() -> None: - if not self.cleared: - self.cleared = True - print("") + def check_repro_impl() -> Tuple[bool, str, bool]: + # check all of the launched repros - commands: Dict[OS, Tuple[str, str]] = { - OS.windows: ("r rip", r"^rip=[a-f0-9]{16}"), - OS.linux: ("info reg rip", r"^rip\s+0x[a-f0-9]+\s+0x[a-f0-9]+"), - } + self.cleared = False - info: Dict[str, List[str]] = {} + def clear() -> None: + if not self.cleared: + self.cleared = True + print("") - done: Set[UUID] = set() - for job_id, repro in self.repros.items(): - repro = self.of.repro.get(repro.vm_id) - if repro.error: - clear() - self.logger.error( - "repro failed: %s: %s", self.target_jobs[job_id], repro.error - ) - self.failed_jobs.add(job_id) - done.add(job_id) - elif repro.state not in [VmState.init, VmState.extensions_launch]: - done.add(job_id) - try: - result = self.of.repro.connect( - repro.vm_id, - delete_after_use=True, - debug_command=commands[repro.os][0], + commands: Dict[OS, Tuple[str, str]] = { + OS.windows: ("r rip", r"^rip=[a-f0-9]{16}"), + OS.linux: ("info reg rip", r"^rip\s+0x[a-f0-9]+\s+0x[a-f0-9]+"), + } + + for (job, repro) in list(repros.values()): + repros[job.job_id] = (job, self.of.repro.get(repro.vm_id)) + + for (job, repro) in list(repros.values()): + if repro.error: + clear() + self.logger.error( + "repro failed: %s: %s", + job.config.name, + repro.error, ) - if result is not None and re.search( - commands[repro.os][1], result, re.MULTILINE - ): - clear() - self.logger.info( - "repro succeeded: %s", self.target_jobs[job_id] + self.of.repro.delete(repro.vm_id) + del repros[job.job_id] + elif repro.state == VmState.running: + try: + result = self.of.repro.connect( + repro.vm_id, + delete_after_use=True, + debug_command=commands[repro.os][0], ) - self.failed_jobs.add(job_id) - done.add(job_id) - else: + if result is not None and re.search( + commands[repro.os][1], result, re.MULTILINE + ): + clear() + self.logger.info("repro succeeded: %s", job.config.name) + else: + clear() + self.logger.error( + "repro failed: %s - %s", job.config.name, result + ) + except Exception as err: clear() - self.logger.error( - "repro failed: %s: %s", self.target_jobs[job_id], result - ) - self.failed_jobs.add(job_id) - done.add(job_id) - except Exception as e: - clear() + self.logger.error("repro failed: %s - %s", job.config.name, err) + del repros[job.job_id] + elif repro.state not in [VmState.init, VmState.extensions_launch]: self.logger.error( - "repro failed: %s: %s", self.target_jobs[job_id], repr(e) + "repro failed: %s - bad state: %s", job.config.name, repro.state ) - self.failed_jobs.add(job_id) - done.add(job_id) - else: - if repro.state.name not in info: - info[repro.state.name] = [] - info[repro.state.name].append(self.target_jobs[job_id]) - - for job_id in done: - self.of.repro.delete(self.repros[job_id].vm_id) - del self.repros[job_id] - - logline = [] - for name in info: - logline.append("%s:%s" % (name, ",".join(info[name]))) - - msg = "waiting repro: %s" % " ".join(logline) - if len(logline) > 80: - msg = "waiting on %d repros" % len(self.repros) - - return ( - not bool(self.repros), - msg, - bool(self.failed_jobs), + del repros[job.job_id] + + repro_states: Dict[str, List[str]] = {} + for (job, repro) in repros.values(): + if repro.state.name not in repro_states: + repro_states[repro.state.name] = [] + repro_states[repro.state.name].append(job.config.name) + + logline = [] + for state in repro_states: + logline.append("%s:%s" % (state, ",".join(repro_states[state]))) + + msg = "waiting repro: %s" % " ".join(logline) + if len(msg) > 80: + msg = "waiting on %d repros" % len(repros) + return (not bool(repros), msg, self.success) + + return wait(check_repro_impl) + + def get_jobs(self) -> List[Job]: + jobs = self.of.jobs.list(job_state=None) + jobs = [x for x in jobs if x.config.project == self.project] + return jobs + + def stop_job(self, job: Job, delete_containers: bool = False) -> None: + self.of.template.stop( + job.config.project, + job.config.name, + BUILD, + delete_containers=delete_containers, ) - def check_repro(self) -> bool: - self.logger.info("checking repros") - return wait(self.check_repro_impl) - - def stop_template(self, target: str, delete_containers: bool = True) -> None: - """ stop a specific template """ + def get_pools(self) -> List[Pool]: + pools = self.of.pools.list() + pools = [x for x in pools if x.name == f"testpool-{x.os.name}-{self.test_id}"] + return pools - if self.skip_cleanup: - self.logger.warning("not cleaning up target: %s", target) - else: - self.of.template.stop( - self.project, - target, - BUILD, - delete_containers=delete_containers, - stop_notifications=True, - ) - - def cleanup(self, *, user_pools: Optional[Dict[str, str]] = None) -> bool: + def cleanup(self) -> None: """ cleanup all of the integration pools & jobs """ - if self.skip_cleanup: - self.logger.warning("not cleaning up") - return True - self.logger.info("cleaning up") errors: List[Exception] = [] - for target, config in TARGETS.items(): - if config.os not in self.os: - continue - if target not in self.targets: - continue - + jobs = self.get_jobs() + for job in jobs: try: - self.logger.info("stopping %s", target) - self.stop_template(target, delete_containers=False) + self.stop_job(job, delete_containers=True) except Exception as e: - self.logger.error("cleanup of %s failed", target) + self.logger.error("cleanup of job failed: %s - %s", job, e) errors.append(e) - for pool in self.pools.values(): - if user_pools and pool.name in user_pools.values(): - continue - + for pool in self.get_pools(): self.logger.info( "halting: %s:%s:%s", pool.name, pool.os.name, pool.arch.name ) @@ -595,52 +615,115 @@ def cleanup(self, *, user_pools: Optional[Dict[str, str]] = None) -> bool: self.logger.error("cleanup of pool failed: %s - %s", pool.name, e) errors.append(e) - for repro in self.repros.values(): - try: - self.of.repro.delete(repro.vm_id) - except Exception as e: - self.logger.error("cleanup of repro failed: %s - %s", repro.vm_id, e) - errors.append(e) + container_names = set() + for job in jobs: + for task in self.of.tasks.list(job_id=job.job_id, state=None): + for container in task.config.containers: + if container.type in [ + ContainerType.reports, + ContainerType.unique_reports, + ]: + container_names.add(container.name) + + for repro in self.of.repro.list(): + if repro.config.container in container_names: + try: + self.of.repro.delete(repro.vm_id) + except Exception as e: + self.logger.error("cleanup of repro failed: %s %s", repro.vm_id, e) + errors.append(e) - return not bool(errors) + if errors: + raise Exception("cleanup failed") class Run(Command): - def test( + def check_jobs( + self, + test_id: UUID, + *, + endpoint: Optional[str], + poll: bool = False, + stop_on_complete_check: bool = False, + ) -> None: + self.onefuzz.__setup__(endpoint=endpoint) + tester = TestOnefuzz(self.onefuzz, self.logger, test_id) + result = tester.check_jobs( + poll=poll, stop_on_complete_check=stop_on_complete_check + ) + if not result: + raise Exception("jobs failed") + + def check_repros(self, test_id: UUID, *, endpoint: Optional[str]) -> None: + self.onefuzz.__setup__(endpoint=endpoint) + tester = TestOnefuzz(self.onefuzz, self.logger, test_id) + launch_result, repros = tester.launch_repro() + result = tester.check_repro(repros) + if not (result and launch_result): + raise Exception("repros failed") + + def launch( self, samples: Directory, *, endpoint: Optional[str] = None, - user_pools: Optional[Dict[str, str]] = None, pool_size: int = 10, region: Optional[Region] = None, os_list: List[OS] = [OS.linux, OS.windows], targets: List[str] = list(TARGETS.keys()), + test_id: Optional[UUID] = None, + duration: int = 1, + ) -> UUID: + if test_id is None: + test_id = uuid4() + self.logger.info("launching test_id: %s", test_id) + + self.onefuzz.__setup__(endpoint=endpoint) + tester = TestOnefuzz(self.onefuzz, self.logger, test_id) + tester.setup(region=region, pool_size=pool_size, os_list=os_list) + tester.launch(samples, os_list=os_list, targets=targets, duration=duration) + return test_id + + def cleanup(self, test_id: UUID, *, endpoint: Optional[str]) -> None: + self.onefuzz.__setup__(endpoint=endpoint) + tester = TestOnefuzz(self.onefuzz, self.logger, test_id=test_id) + tester.cleanup() + + def test( + self, + samples: Directory, + *, + endpoint: Optional[str] = None, + pool_size: int = 15, + region: Optional[Region] = None, + os_list: List[OS] = [OS.linux, OS.windows], + targets: List[str] = list(TARGETS.keys()), skip_repro: bool = False, - skip_cleanup: bool = False, + duration: int = 1, ) -> None: - self.onefuzz.__setup__(endpoint=endpoint) - tester = TestOnefuzz( - self.onefuzz, - self.logger, - pool_size=pool_size, - os_list=os_list, - targets=targets, - skip_cleanup=skip_cleanup, - ) success = True + test_id = uuid4() error: Optional[Exception] = None try: - tester.setup(region=region, user_pools=user_pools) - tester.launch(samples) - tester.check_jobs() + self.launch( + samples, + endpoint=endpoint, + pool_size=pool_size, + region=region, + os_list=os_list, + targets=targets, + test_id=test_id, + duration=duration, + ) + self.check_jobs( + test_id, endpoint=endpoint, poll=True, stop_on_complete_check=True + ) + if skip_repro: self.logger.warning("not testing crash repro") else: - self.logger.info("launching crash repro tests") - tester.launch_repro() - tester.check_repro() + self.check_repros(test_id, endpoint=endpoint) except Exception as e: self.logger.error("testing failed: %s", repr(e)) error = e @@ -649,10 +732,11 @@ def test( self.logger.error("interrupted testing") success = False - if not tester.cleanup(user_pools=user_pools): - success = False - - if tester.failed_jobs or tester.failed_repro: + try: + self.cleanup(test_id, endpoint=endpoint) + except Exception as e: + self.logger.error("testing failed: %s", repr(e)) + error = e success = False if error: From 54e007045ec58cdb5e6b3c18c39a6ae5b3ffe0f5 Mon Sep 17 00:00:00 2001 From: Jason Shirk Date: Wed, 10 Mar 2021 14:52:17 -0800 Subject: [PATCH 03/13] Fix incorrect offset in stack reports (#658) --- src/agent/debugger/src/dbghelp.rs | 4 ++-- src/agent/debugger/src/debugger.rs | 2 +- src/agent/debugger/src/stack.rs | 6 ++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/agent/debugger/src/dbghelp.rs b/src/agent/debugger/src/dbghelp.rs index 52d4b0a5c0..7b960faf33 100644 --- a/src/agent/debugger/src/dbghelp.rs +++ b/src/agent/debugger/src/dbghelp.rs @@ -514,7 +514,7 @@ impl DebugHelpGuard { } } - pub fn stackwalk_ex bool>( + pub fn stackwalk_ex bool>( &self, process_handle: HANDLE, thread_handle: HANDLE, @@ -550,7 +550,7 @@ impl DebugHelpGuard { break; } - if !f(&frame_context, &frame) { + if !f(&frame) { break; } } diff --git a/src/agent/debugger/src/debugger.rs b/src/agent/debugger/src/debugger.rs index 800cd10df1..cafbd6f1d3 100644 --- a/src/agent/debugger/src/debugger.rs +++ b/src/agent/debugger/src/debugger.rs @@ -699,7 +699,7 @@ impl Debugger { dbghlp.stackwalk_ex( self.target.process_handle(), self.target.current_thread_handle(), - |_frame_context, frame| { + |frame| { return_address = frame.AddrReturn; stack_pointer = frame.AddrStack; diff --git a/src/agent/debugger/src/stack.rs b/src/agent/debugger/src/stack.rs index 637a7b57d8..17e083912e 100644 --- a/src/agent/debugger/src/stack.rs +++ b/src/agent/debugger/src/stack.rs @@ -223,10 +223,8 @@ pub fn get_stack( let mut stack = vec![]; - dbghlp.stackwalk_ex(process_handle, thread_handle, |frame_context, frame| { - // The program counter is the return address, potentially outside of the function - // performing the call. We subtract 1 to ensure the address is within the call. - let program_counter = frame_context.program_counter().saturating_sub(1); + dbghlp.stackwalk_ex(process_handle, thread_handle, |frame| { + let program_counter = frame.AddrPC.Offset; let debug_stack_frame = if resolve_symbols { if let Ok(module_info) = dbghlp.sym_get_module_info(process_handle, program_counter) { From c429787d7275c8a13dd19d5a8eacb27075c8e329 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 10 Mar 2021 18:01:49 -0500 Subject: [PATCH 04/13] add integration tests that verify regression tasks --- src/integration-tests/check-regression.py | 157 ++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 src/integration-tests/check-regression.py diff --git a/src/integration-tests/check-regression.py b/src/integration-tests/check-regression.py new file mode 100644 index 0000000000..294bfd72b4 --- /dev/null +++ b/src/integration-tests/check-regression.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# + +import json +import logging +import os +import sys +import time +from typing import Optional +from uuid import UUID, uuid4 + +from onefuzz.api import Command, Onefuzz +from onefuzz.cli import execute_api +from onefuzztypes.enums import OS, ContainerType, TaskState, TaskType +from onefuzztypes.models import Job, RegressionReport +from onefuzztypes.primitives import Container, Directory, File, PoolName + + +class Run(Command): + def cleanup(self, test_id: UUID): + for pool in self.onefuzz.pools.list(): + if str(test_id) in pool.name: + self.onefuzz.pools.shutdown(pool.name, now=True) + + self.onefuzz.template.stop( + str(test_id), "linux-libfuzzer", build=None, delete_containers=True + ) + + def _wait_for_regression_task(self, job: Job) -> None: + while True: + self.logger.info("waiting for regression task to finish") + for task in self.onefuzz.jobs.tasks.list(job.job_id): + if task.config.task.type not in [ + TaskType.libfuzzer_regression, + TaskType.generic_regression, + ]: + continue + if task.state != TaskState.stopped: + continue + return + time.sleep(10) + + def _check_regression(self, job: Job) -> bool: + # get the regression reports containers for the job + results = self.onefuzz.jobs.containers.list( + job.job_id, ContainerType.regression_reports + ) + + # expect one and only one regression report container + if len(results) != 1: + raise Exception(f"unexpected regression containers: {results}") + container = list(results.keys())[0] + + # expect one and only one file in the container + if len(results[container]) != 1: + raise Exception(f"unexpected regression container output: {results}") + file = results[container][0] + + # get the regression report + content = self.onefuzz.containers.files.get(Container(container), file) + as_str = content.decode() + as_obj = json.loads(as_str) + report = RegressionReport.parse_obj(as_obj) + + if report.crash_test_result.crash_report is not None: + self.logger.info("regression report has crash report") + return True + + if report.crash_test_result.no_repro is not None: + self.logger.info("regression report has no-repro") + return False + + raise Exception(f"unexpected report: {report}") + + + def _run_job( + self, test_id: UUID, pool: PoolName, target: str, exe: File, build: int + ) -> Job: + if build == 1: + wait_for_files = [ContainerType.unique_reports] + else: + wait_for_files = [ContainerType.regression_reports] + job = self.onefuzz.template.libfuzzer.basic( + str(test_id), + target, + str(build), + pool, + target_exe=exe, + duration=1, + vm_count=1, + wait_for_files=wait_for_files, + ) + if job is None: + raise Exception(f"invalid job: {target} {build}") + + if build > 1: + self._wait_for_regression_task(job) + self.onefuzz.template.stop(str(test_id), target, str(build)) + return job + + def _run(self, target_os: OS, test_id: UUID, base: Directory, target: str) -> None: + pool = PoolName(f"{target}-{target_os.name}-{test_id}") + self.onefuzz.pools.create(pool, target_os) + self.onefuzz.scalesets.create(pool, 5) + broken = File(os.path.join(base, target, "broken.exe")) + fixed = File(os.path.join(base, target, "fixed.exe")) + + self.logger.info("starting first build") + self._run_job(test_id, pool, target, broken, 1) + + self.logger.info("starting second build") + job = self._run_job(test_id, pool, target, fixed, 2) + if self._check_regression(job): + raise Exception("fixed binary should be a no repro") + + self.logger.info("starting third build") + job = self._run_job(test_id, pool, target, broken, 3) + if not self._check_regression(job): + raise Exception("broken binary should be a crash report") + + self.onefuzz.pools.shutdown(pool, now=True) + + def test( + self, + samples: Directory, + *, + endpoint: Optional[str] = None, + ): + test_id = uuid4() + self.logger.info(f"launch test {test_id}") + self.onefuzz.__setup__(endpoint=endpoint) + error: Optional[Exception] = None + try: + self._run(OS.linux, test_id, samples, "linux-libfuzzer-regression") + except Exception as err: + error = err + except KeyboardInterrupt: + self.logger.warning("interruptted") + finally: + self.logger.info("cleaning up tests") + self.cleanup(test_id) + + if error: + raise error + + +def main() -> int: + return execute_api( + Run(Onefuzz(), logging.getLogger("regression")), [Command], "0.0.1" + ) + + +if __name__ == "__main__": + sys.exit(main()) From 4a8b8759965384e3953ac8082496a50342fedf20 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 10 Mar 2021 23:20:28 -0500 Subject: [PATCH 05/13] readonly_inputs needs to be init_pull, not sync_pull --- src/agent/onefuzz-agent/src/tasks/regression/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/onefuzz-agent/src/tasks/regression/common.rs b/src/agent/onefuzz-agent/src/tasks/regression/common.rs index e75af1e57b..ac373b6c05 100644 --- a/src/agent/onefuzz-agent/src/tasks/regression/common.rs +++ b/src/agent/onefuzz-agent/src/tasks/regression/common.rs @@ -69,7 +69,7 @@ pub async fn handle_inputs( regression_reports: &SyncedDir, heartbeat_client: &Option, ) -> Result<()> { - readonly_inputs.sync_pull().await?; + readonly_inputs.init_pull().await?; let mut input_files = tokio::fs::read_dir(&readonly_inputs.path).await?; while let Some(file) = input_files.next_entry().await? { heartbeat_client.alive(); From 41c280bc1d68f0012b129ad3b86e534729c0d9c6 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 10 Mar 2021 23:21:49 -0500 Subject: [PATCH 06/13] cleanup bisect --- src/cli/onefuzz/job_templates/job_monitor.py | 14 +-- src/cli/onefuzz/templates/__init__.py | 4 +- src/cli/onefuzz/templates/regression.py | 98 ++++++++++++-------- src/integration-tests/check-regression.py | 5 +- 4 files changed, 69 insertions(+), 52 deletions(-) diff --git a/src/cli/onefuzz/job_templates/job_monitor.py b/src/cli/onefuzz/job_templates/job_monitor.py index f0d497faa7..f0bb11f708 100644 --- a/src/cli/onefuzz/job_templates/job_monitor.py +++ b/src/cli/onefuzz/job_templates/job_monitor.py @@ -78,21 +78,21 @@ def has_files(self) -> Tuple[bool, str, Any]: None, ) - def is_stopping(self) -> Tuple[bool, str, Any]: + def is_stopped(self) -> Tuple[bool, str, Any]: tasks = self.onefuzz.tasks.list(job_id=self.job.job_id) - stopping = [ + waiting = [ "%s:%s" % (x.config.task.type.name, x.state.name) for x in tasks - if x.state not in TaskState.shutting_down() + if x.state != TaskState.stopped ] - return (not stopping, "waiting on: %s" % ", ".join(sorted(stopping)), None) + return (not waiting, "waiting on: %s" % ", ".join(sorted(waiting)), None) def wait( self, *, wait_for_running: Optional[bool] = False, wait_for_files: Optional[Dict[Container, int]] = None, - wait_for_stopping: Optional[bool] = False, + wait_for_stopped: Optional[bool] = False, ) -> None: if wait_for_running: wait(self.is_running) @@ -103,6 +103,6 @@ def wait( wait(self.has_files) self.onefuzz.logger.info("new files found") - if wait_for_stopping: - wait(self.is_stopping) + if wait_for_stopped: + wait(self.is_stopped) self.onefuzz.logger.info("tasks stopped") diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index 886b56cacd..e2b548d00c 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -94,7 +94,7 @@ def __init__( ) self.wait_for_running: bool = False - self.wait_for_stopping: bool = False + self.wait_for_stopped: bool = False self.containers: Dict[ContainerType, Container] = {} self.tags: Dict[str, str] = {"project": project, "name": name, "build": build} if job is None: @@ -304,7 +304,7 @@ def wait(self) -> None: JobMonitor(self.onefuzz, self.job).wait( wait_for_running=self.wait_for_running, wait_for_files=self.to_monitor, - wait_for_stopping=self.wait_for_stopping, + wait_for_stopped=self.wait_for_stopped, ) def target_exe_blob_name( diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index 6a5c166efb..80260d2b2e 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -3,11 +3,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import json from typing import Dict, List, Optional from onefuzztypes.enums import ContainerType, TaskDebugFlag, TaskType -from onefuzztypes.models import Job, NotificationConfig -from onefuzztypes.primitives import Directory, File, PoolName +from onefuzztypes.models import Job, NotificationConfig, RegressionReport +from onefuzztypes.primitives import Container, Directory, File, PoolName from onefuzz.api import Command @@ -22,6 +23,22 @@ class Regression(Command): """ Regression job """ + def _check_regression(self, container: Container, file: File) -> bool: + content = self.onefuzz.containers.files.get(Container(container), file) + as_str = content.decode() + as_obj = json.loads(as_str) + report = RegressionReport.parse_obj(as_obj) + + if report.crash_test_result.crash_report is not None: + self.logger.error("regression: %s %s", container, file) + return True + + if report.crash_test_result.no_repro is not None: + self.logger.info("no repro: %s %s", container, file) + return False + + raise Exception("invalid crash report") + def generic( self, project: str, @@ -44,15 +61,15 @@ def generic( debug: Optional[List[TaskDebugFlag]] = None, check_retry_count: Optional[int] = None, check_fuzzer_help: bool = True, - fail_on_repro: bool = False, delete_input_container: bool = True, + check_regressions: bool = False, ) -> Optional[Job]: """ generic regression task :param File inputs: Specify a directory of inptus to use in the regression task :param str reports: Specify specific report names to verify in the regression task - :param bool fail_on_repro: Specify wether or not to throw an exception if a repro was generated + :param bool check_regressions: Specify if exceptions shoudl be through on finding crash regressions :param bool delete_input_container: Specify wether or not to delete the input container """ @@ -77,8 +94,8 @@ def generic( debug=debug, check_retry_count=check_retry_count, check_fuzzer_help=check_fuzzer_help, - fail_on_repro=fail_on_repro, delete_input_container=delete_input_container, + check_regressions=check_regressions, ) def libfuzzer( @@ -103,8 +120,8 @@ def libfuzzer( debug: Optional[List[TaskDebugFlag]] = None, check_retry_count: Optional[int] = None, check_fuzzer_help: bool = True, - fail_on_repro: bool = False, delete_input_container: bool = True, + check_regressions: bool = False, ) -> Optional[Job]: """ @@ -112,7 +129,7 @@ def libfuzzer( :param File inputs: Specify a directory of inptus to use in the regression task :param str reports: Specify specific report names to verify in the regression task - :param bool fail_on_repro: Specify wether or not to throw an exception if a repro was generated + :param bool check_regressions: Specify if exceptions shoudl be through on finding crash regressions :param bool delete_input_container: Specify wether or not to delete the input container """ @@ -137,8 +154,8 @@ def libfuzzer( debug=debug, check_retry_count=check_retry_count, check_fuzzer_help=check_fuzzer_help, - fail_on_repro=fail_on_repro, delete_input_container=delete_input_container, + check_regressions=check_regressions, ) def _create_job( @@ -164,8 +181,8 @@ def _create_job( debug: Optional[List[TaskDebugFlag]] = None, check_retry_count: Optional[int] = None, check_fuzzer_help: bool = True, - fail_on_repro: bool = False, delete_input_container: bool = True, + check_regressions: bool = False, ) -> Optional[Job]: if dryrun: @@ -193,20 +210,6 @@ def _create_job( ContainerType.regression_reports, ) - if crashes: - helper.containers[ - ContainerType.readonly_inputs - ] = helper.get_unique_container_name(ContainerType.readonly_inputs) - - helper.create_containers() - if crashes: - for file in crashes: - self.onefuzz.containers.files.upload_file( - helper.containers[ContainerType.unique_inputs], file - ) - - helper.setup_notifications(notification_config) - containers = [ (ContainerType.setup, helper.containers[ContainerType.setup]), (ContainerType.crashes, helper.containers[ContainerType.crashes]), @@ -222,11 +225,31 @@ def _create_job( ), ] + if crashes: + helper.containers[ + ContainerType.readonly_inputs + ] = helper.get_unique_container_name(ContainerType.readonly_inputs) + containers.append( + ( + ContainerType.readonly_inputs, + helper.containers[ContainerType.readonly_inputs], + ) + ) + + helper.create_containers() + if crashes: + for file in crashes: + self.onefuzz.containers.files.upload_file( + helper.containers[ContainerType.readonly_inputs], file + ) + + helper.setup_notifications(notification_config) + helper.upload_setup(setup_dir, target_exe) target_exe_blob_name = helper.target_exe_blob_name(target_exe, setup_dir) self.logger.info("creating regression task") - regression_task = self.onefuzz.tasks.create( + task = self.onefuzz.tasks.create( helper.job.job_id, task_type, target_exe_blob_name, @@ -244,27 +267,22 @@ def _create_job( check_fuzzer_help=check_fuzzer_help, report_list=reports, ) - helper.wait_for_stopping = fail_on_repro + helper.wait_for_stopped = check_regressions self.logger.info("done creating tasks") helper.wait() - if fail_on_repro: - if helper.job.error: - raise TemplateError( - "unable to run the the regression", GIT_BISSECT_SKIP_CODE - ) + if check_regressions: + task = self.onefuzz.tasks.get(task.task_id) + if task.error: + raise Exception("task failed: %s", task.error) - repro_count = len( - self.onefuzz.containers.files.list( - helper.containers[ContainerType.regression_reports], - prefix=str(regression_task.task_id), - ).files - ) - if repro_count > 0: - raise TemplateError("Failure detected", -1) - else: - self.logger.info("No Failure detected") + container = helper.containers[ContainerType.regression_reports] + for filename in self.onefuzz.containers.files.list(container).files: + self.logger.info("checking file: %s", filename) + if self._check_regression(container, File(filename)): + raise TemplateError("Failure detected", -1) + self.logger.info("no regressions") if ( delete_input_container diff --git a/src/integration-tests/check-regression.py b/src/integration-tests/check-regression.py index 294bfd72b4..2ae0d76723 100644 --- a/src/integration-tests/check-regression.py +++ b/src/integration-tests/check-regression.py @@ -54,7 +54,7 @@ def _check_regression(self, job: Job) -> bool: raise Exception(f"unexpected regression containers: {results}") container = list(results.keys())[0] - # expect one and only one file in the container + # expect one and only one file in the container if len(results[container]) != 1: raise Exception(f"unexpected regression container output: {results}") file = results[container][0] @@ -68,14 +68,13 @@ def _check_regression(self, job: Job) -> bool: if report.crash_test_result.crash_report is not None: self.logger.info("regression report has crash report") return True - + if report.crash_test_result.no_repro is not None: self.logger.info("regression report has no-repro") return False raise Exception(f"unexpected report: {report}") - def _run_job( self, test_id: UUID, pool: PoolName, target: str, exe: File, build: int ) -> Job: From 867e9b6208b9c22e983fadf57dda0735bbb167f8 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 10 Mar 2021 23:23:00 -0500 Subject: [PATCH 07/13] add examples for integration tests --- src/integration-tests/git-bisect/README.md | 21 ++++++++++++++++ src/integration-tests/git-bisect/build.sh | 15 ++++++++++++ src/integration-tests/git-bisect/run-local.sh | 23 ++++++++++++++++++ .../git-bisect/run-onefuzz.sh | 24 +++++++++++++++++++ src/integration-tests/git-bisect/src/Makefile | 13 ++++++++++ .../git-bisect/src/bisect-local.sh | 7 ++++++ .../git-bisect/src/bisect-onefuzz.sh | 16 +++++++++++++ src/integration-tests/git-bisect/src/fuzz.c | 13 ++++++++++ 8 files changed, 132 insertions(+) create mode 100644 src/integration-tests/git-bisect/README.md create mode 100755 src/integration-tests/git-bisect/build.sh create mode 100755 src/integration-tests/git-bisect/run-local.sh create mode 100755 src/integration-tests/git-bisect/run-onefuzz.sh create mode 100644 src/integration-tests/git-bisect/src/Makefile create mode 100755 src/integration-tests/git-bisect/src/bisect-local.sh create mode 100755 src/integration-tests/git-bisect/src/bisect-onefuzz.sh create mode 100644 src/integration-tests/git-bisect/src/fuzz.c diff --git a/src/integration-tests/git-bisect/README.md b/src/integration-tests/git-bisect/README.md new file mode 100644 index 0000000000..eabf5f0bc7 --- /dev/null +++ b/src/integration-tests/git-bisect/README.md @@ -0,0 +1,21 @@ +# git-bisect regression source + +This assumes you have a working clang with libfuzzer, bash, and git. + +This makes a git repo `test` with 9 commits. Each commit after the first adds a bug. + +* `commit 0` has no bugs. +* `commit 1` will additionally cause an abort if the input is `1`. +* `commit 2` will additionally cause an abort if the input is `2`. +* `commit 3` will additionally cause an abort if the input is `3`. +* etc. + +This directory provides exemplar scripts that demonstrate how to perform + `git bisect` with libfuzzer. + + * [run-local.sh](run-local.sh) builds & runs the libfuzzer target locally. It uses [src/bisect-local.sh](src/bisect-local.sh) as the `git bisect run` command. + * [run-onefuzz.sh](run-onefuzz.sh) builds the libfuzzer target locally, but uses OneFuzz to run the regression tasks. It uses [src/bisect-onefuzz.sh](src/bisect-onefuzz.sh) as the `git bisect run` command. + +With each project having their own unique paradigm for building, this model +allows plugging OneFuzz as a `bisect` command in whatever fashion your +project requires. \ No newline at end of file diff --git a/src/integration-tests/git-bisect/build.sh b/src/integration-tests/git-bisect/build.sh new file mode 100755 index 0000000000..d0f47c3f4d --- /dev/null +++ b/src/integration-tests/git-bisect/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +git init test +(cp src/Makefile test; cd test; git add Makefile) +for i in $(seq 0 8); do + cp src/fuzz.c test/fuzz.c + for j in $(seq $i 8); do + if [ $i != $j ]; then + sed -i /TEST$j/d test/fuzz.c + fi + done + (cd test; git add fuzz.c; git commit -m "commit $i") +done \ No newline at end of file diff --git a/src/integration-tests/git-bisect/run-local.sh b/src/integration-tests/git-bisect/run-local.sh new file mode 100755 index 0000000000..d67b0b705f --- /dev/null +++ b/src/integration-tests/git-bisect/run-local.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -ex + +# build our git repo with our samples in `test` +./build.sh + +# create our crashing input +echo -n '3' > test/test.txt + +cd test + +# start the bisect, looking from HEAD backwards 8 commits +git bisect start HEAD HEAD~8 -- +# run the local bisect tool, checking if we crash locally +git bisect run ../src/bisect-local.sh test.txt + +# print the current commit (if this works, we expect to see 'commit 3') +echo "The next line should show 'commit 3'" +git show -s --format=%s + +# reset git back the state it was prior to the bisect. +git bisect reset \ No newline at end of file diff --git a/src/integration-tests/git-bisect/run-onefuzz.sh b/src/integration-tests/git-bisect/run-onefuzz.sh new file mode 100755 index 0000000000..cb2d412544 --- /dev/null +++ b/src/integration-tests/git-bisect/run-onefuzz.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -ex + +# build our git repo with our samples in `test` +./build.sh + +# create our crashing input +echo -n '3' > test/test.txt + +cd test + +# start the bisect, looking from HEAD backwards 8 commits +git bisect start HEAD HEAD~8 -- + +# run the local bisect tool, checking if we crash locally +git bisect run ../src/bisect-onefuzz.sh test.txt + +# print the current commit (if this works, we expect to see 'commit 3') +echo "The next line should show 'commit 3'" +git show -s --format=%s + +# reset git back the state it was prior to the bisect. +git bisect reset \ No newline at end of file diff --git a/src/integration-tests/git-bisect/src/Makefile b/src/integration-tests/git-bisect/src/Makefile new file mode 100644 index 0000000000..a3a273cafb --- /dev/null +++ b/src/integration-tests/git-bisect/src/Makefile @@ -0,0 +1,13 @@ +CC=clang + +CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address + +all: fuzz.exe + +fuzz.exe: fuzz.c + $(CC) $(CFLAGS) fuzz.c -o fuzz.exe + +.PHONY: clean + +clean: + rm -f fuzz.exe diff --git a/src/integration-tests/git-bisect/src/bisect-local.sh b/src/integration-tests/git-bisect/src/bisect-local.sh new file mode 100755 index 0000000000..69d43001f7 --- /dev/null +++ b/src/integration-tests/git-bisect/src/bisect-local.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -ex + +make clean +make +./fuzz.exe $* \ No newline at end of file diff --git a/src/integration-tests/git-bisect/src/bisect-onefuzz.sh b/src/integration-tests/git-bisect/src/bisect-onefuzz.sh new file mode 100755 index 0000000000..727735e1f1 --- /dev/null +++ b/src/integration-tests/git-bisect/src/bisect-onefuzz.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -ex + +PROJECT=${PROJECT:-regression-test} +TARGET=${TARGET:-$(uuidgen)} +BUILD=regression-$(git rev-parse HEAD) +POOL=${ONEFUZZ_POOL:-linux} + +echo 'checking build' +git show -s --format=%s + +make clean +make +onefuzz template regression libfuzzer ${PROJECT} ${TARGET} ${BUILD} ${POOL} --check_regressions --delete_input_container --reports --crashes $* +echo 'not this onefuzz job' \ No newline at end of file diff --git a/src/integration-tests/git-bisect/src/fuzz.c b/src/integration-tests/git-bisect/src/fuzz.c new file mode 100644 index 0000000000..53c2cd1cb5 --- /dev/null +++ b/src/integration-tests/git-bisect/src/fuzz.c @@ -0,0 +1,13 @@ +#include +int LLVMFuzzerTestOneInput(char *data, size_t len) { + if (len != 1) { return 0; } + if (data[0] == '1') { abort(); } // TEST1 + if (data[0] == '2') { abort(); } // TEST2 + if (data[0] == '3') { abort(); } // TEST3 + if (data[0] == '4') { abort(); } // TEST4 + if (data[0] == '5') { abort(); } // TEST5 + if (data[0] == '6') { abort(); } // TEST6 + if (data[0] == '7') { abort(); } // TEST7 + if (data[0] == '8') { abort(); } // TEST8 + return 0; +} From 0fcee90b62916d44d811664c047bd6cdcc4abd17 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 11 Mar 2021 00:58:09 -0500 Subject: [PATCH 08/13] log slightly less --- src/cli/onefuzz/templates/regression.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index 80260d2b2e..5bbb32dd82 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -30,11 +30,9 @@ def _check_regression(self, container: Container, file: File) -> bool: report = RegressionReport.parse_obj(as_obj) if report.crash_test_result.crash_report is not None: - self.logger.error("regression: %s %s", container, file) return True if report.crash_test_result.no_repro is not None: - self.logger.info("no repro: %s %s", container, file) return False raise Exception("invalid crash report") @@ -281,7 +279,7 @@ def _create_job( for filename in self.onefuzz.containers.files.list(container).files: self.logger.info("checking file: %s", filename) if self._check_regression(container, File(filename)): - raise TemplateError("Failure detected", -1) + raise Exception("regression identified: %s", filename) self.logger.info("no regressions") if ( From 638cb0f914618e129041b3d9b9725b0fdad51be8 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 11 Mar 2021 01:00:48 -0500 Subject: [PATCH 09/13] not using special exit code to indicate failures --- src/cli/onefuzz/templates/regression.py | 5 ----- src/cli/onefuzz/templates/template_error.py | 11 ----------- 2 files changed, 16 deletions(-) delete mode 100644 src/cli/onefuzz/templates/template_error.py diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index 5bbb32dd82..5c3f6dcbfb 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -13,11 +13,6 @@ from onefuzz.api import Command from . import JobHelper -from .template_error import TemplateError - -# Special exit code value used by git bisect to determine if a commit should be skipped -# https://git-scm.com/docs/git-bisect#_bisect_run -GIT_BISSECT_SKIP_CODE = 125 class Regression(Command): diff --git a/src/cli/onefuzz/templates/template_error.py b/src/cli/onefuzz/templates/template_error.py deleted file mode 100644 index ae817ea9eb..0000000000 --- a/src/cli/onefuzz/templates/template_error.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - - -class TemplateError(Exception): - def __init__(self, message: str, status_code: int) -> None: - super(TemplateError, self).__init__(message) - self.message = message - self.status_code = status_code From 9ef2b999a6bf65caa0ba311a6d7072553461394e Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 11 Mar 2021 01:06:35 -0500 Subject: [PATCH 10/13] remove TemplateError use from cli.py --- src/cli/onefuzz/cli.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/cli/onefuzz/cli.py b/src/cli/onefuzz/cli.py index 7feaad70ec..8d4028e22c 100644 --- a/src/cli/onefuzz/cli.py +++ b/src/cli/onefuzz/cli.py @@ -35,7 +35,6 @@ from onefuzztypes.primitives import Container, Directory, File, PoolName, Region from pydantic import BaseModel, ValidationError -from .templates.template_error import TemplateError LOGGER = logging.getLogger("cli") @@ -554,10 +553,6 @@ def execute_api(api: Any, api_types: List[Any], version: str) -> int: try: result = call_func(args.func, args) - except TemplateError as err: - LOGGER.error(err.message) - log_exception(args, err) - return err.status_code except Exception as err: log_exception(args, err) return 1 From e829e9f8f434e8d805c3a4f2780ddfa5e273cabe Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 11 Mar 2021 01:33:44 -0500 Subject: [PATCH 11/13] normalize file path generation --- .../src/tasks/regression/common.rs | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/agent/onefuzz-agent/src/tasks/regression/common.rs b/src/agent/onefuzz-agent/src/tasks/regression/common.rs index ac373b6c05..60a7602efb 100644 --- a/src/agent/onefuzz-agent/src/tasks/regression/common.rs +++ b/src/agent/onefuzz-agent/src/tasks/regression/common.rs @@ -73,13 +73,24 @@ pub async fn handle_inputs( let mut input_files = tokio::fs::read_dir(&readonly_inputs.path).await?; while let Some(file) = input_files.next_entry().await? { heartbeat_client.alive(); - let input_url = readonly_inputs.url.clone().and_then(|container_url| { - let os_file_name = file.file_name(); - let file_name = os_file_name.to_str()?; - container_url.url().join(file_name).ok() - }); - let crash_test_result = handler.get_crash_result(file.path(), input_url).await?; + let file_path = file.path(); + if !file_path.is_file() { + continue; + } + + let file_name = file_path + .file_name() + .ok_or_else(|| format_err!("missing filename"))? + .to_string_lossy() + .to_string(); + + let input_url = readonly_inputs + .url + .as_ref() + .and_then(|container_url| container_url.url().join(&file_name).ok()); + + let crash_test_result = handler.get_crash_result(file_path, input_url).await?; RegressionReport { crash_test_result, original_crash_test_result: None, From 277725776d5c80d2086d12110cf567d0456a3656 Mon Sep 17 00:00:00 2001 From: bmc-msft <41130664+bmc-msft@users.noreply.github.com> Date: Thu, 11 Mar 2021 01:38:42 -0500 Subject: [PATCH 12/13] automatic retry on wget failure (#659) --- src/ci/azcopy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/azcopy.sh b/src/ci/azcopy.sh index 370f6f1b7b..39c6330247 100755 --- a/src/ci/azcopy.sh +++ b/src/ci/azcopy.sh @@ -7,10 +7,10 @@ set -ex mkdir -p artifacts/azcopy -wget -O azcopy.zip https://aka.ms/downloadazcopy-v10-windows +wget --retry-connrefused -t 30 --waitretry=5 -O azcopy.zip https://aka.ms/downloadazcopy-v10-windows unzip azcopy.zip mv azcopy_windows*/* artifacts/azcopy/ -wget -O azcopy.tgz https://aka.ms/downloadazcopy-v10-linux +wget --retry-connrefused -t 30 --waitretry=5 -O azcopy.tgz https://aka.ms/downloadazcopy-v10-linux tar zxvf azcopy.tgz mv azcopy_linux_amd64*/* artifacts/azcopy/ From 9b3ebfe526345e4ddeead1a22a6d65df8ee910cb Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 11 Mar 2021 01:40:01 -0500 Subject: [PATCH 13/13] lint --- src/cli/onefuzz/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cli/onefuzz/cli.py b/src/cli/onefuzz/cli.py index 8d4028e22c..4e557c650d 100644 --- a/src/cli/onefuzz/cli.py +++ b/src/cli/onefuzz/cli.py @@ -35,7 +35,6 @@ from onefuzztypes.primitives import Container, Directory, File, PoolName, Region from pydantic import BaseModel, ValidationError - LOGGER = logging.getLogger("cli") JMES_HELP = (