From b84896802c75724ab1d475b25b8d40f2f84118db Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Thu, 23 Feb 2023 11:08:01 -0800 Subject: [PATCH] Adding extra container to tasks (#2847) * adding extra container to tasks * setup expand * build fix * generate docs * build fix * build fix * build fix * format * format * build fix * fix extra container references * format * Update "Needs Triage" label to the one we use. (#2845) * Report extension errors (#2846) Old failure message: ``` failed to launch extension ``` New failure message: ``` failed to launch extension(s): Errors for extension 'CustomScriptExtension': :Error: ProvisioningState/failed/3 (Provisioning failed) - Failed to download all specified files. Exiting. Error Message: The remote server returned an error: (400) Bad Request. ``` * Sematically validate notification configs (#2850) * Add new command * Update remaining jinja templates and references to use scriban * Add ado template validation * Validate ado and github templates * Remove unnecessary function * Update src/ApiService/ApiService/OneFuzzTypes/Model.cs Co-authored-by: Cheick Keita --------- Co-authored-by: Cheick Keita * adding extra container to integration tests * adding doc * update tests * format * build and clippy fix * Update src/agent/onefuzz-task/src/tasks/report/generic.rs Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com> --------- Co-authored-by: Marc Greisen Co-authored-by: George Pollard Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com> --- docs/command-replacements.md | 1 + docs/webhook_events.md | 24 +++-- .../ApiService/OneFuzzTypes/Enums.cs | 3 +- .../ApiService/OneFuzzTypes/Model.cs | 2 + .../ApiService/onefuzzlib/Config.cs | 3 + src/ApiService/ApiService/onefuzzlib/Defs.cs | 98 ++++++++++++++++++- .../ApiService/onefuzzlib/Scheduler.cs | 7 +- src/agent/coverage/src/binary.rs | 2 +- src/agent/onefuzz-agent/src/agent/tests.rs | 1 + src/agent/onefuzz-agent/src/debug.rs | 7 +- src/agent/onefuzz-agent/src/scheduler.rs | 3 +- src/agent/onefuzz-agent/src/setup.rs | 14 +++ src/agent/onefuzz-agent/src/work.rs | 16 +++ src/agent/onefuzz-agent/src/worker.rs | 31 +++++- src/agent/onefuzz-agent/src/worker/double.rs | 7 +- src/agent/onefuzz-agent/src/worker/tests.rs | 9 +- src/agent/onefuzz-task/src/local/common.rs | 3 + .../src/local/libfuzzer_test_input.rs | 2 + .../onefuzz-task/src/local/test_input.rs | 1 + src/agent/onefuzz-task/src/managed/cmd.rs | 8 +- .../src/tasks/analysis/generic.rs | 3 + src/agent/onefuzz-task/src/tasks/config.rs | 7 +- .../onefuzz-task/src/tasks/coverage/dotnet.rs | 3 + .../src/tasks/coverage/generic.rs | 3 + .../onefuzz-task/src/tasks/fuzz/generator.rs | 5 + .../src/tasks/fuzz/libfuzzer/dotnet.rs | 1 + .../src/tasks/fuzz/libfuzzer/generic.rs | 1 + .../onefuzz-task/src/tasks/fuzz/supervisor.rs | 4 + .../onefuzz-task/src/tasks/merge/generic.rs | 3 + .../src/tasks/merge/libfuzzer_merge.rs | 2 + .../src/tasks/regression/generic.rs | 2 + .../src/tasks/regression/libfuzzer.rs | 1 + .../src/tasks/report/dotnet/generic.rs | 5 +- .../onefuzz-task/src/tasks/report/generic.rs | 5 + .../src/tasks/report/libfuzzer_report.rs | 4 + src/agent/onefuzz/examples/test-input.rs | 1 + src/agent/onefuzz/src/expand.rs | 8 ++ src/agent/onefuzz/src/input_tester.rs | 8 +- src/agent/onefuzz/src/libfuzzer.rs | 9 ++ src/cli/onefuzz/templates/afl.py | 12 +++ src/cli/onefuzz/templates/libfuzzer.py | 31 +++++- src/cli/onefuzz/templates/ossfuzz.py | 7 +- src/cli/onefuzz/templates/radamsa.py | 11 +++ src/cli/onefuzz/templates/regression.py | 8 ++ src/integration-tests/integration-test.py | 6 ++ src/pytypes/onefuzztypes/enums.py | 1 + src/pytypes/onefuzztypes/models.py | 1 + 47 files changed, 367 insertions(+), 27 deletions(-) diff --git a/docs/command-replacements.md b/docs/command-replacements.md index 7ee7eb9f4b..d3dd09afb7 100644 --- a/docs/command-replacements.md +++ b/docs/command-replacements.md @@ -26,6 +26,7 @@ The following values are replaced with the specific values at runtime. * `{crashes_container}`: Container name for the `crashes` container * `{microsoft_telemetry_key}`: Application Insights key used for collecting [non-attributable telemetry](telemetry.md) to improve OneFuzz. * `{instance_telemetry_key}`: Application Insights key used for private, instance-owned telemetry and logging (See [OneFuzz Telemetry](telemetry.md). +* `{extra}`: Path to the optionally provided `extra` directory ## Example diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 554d1144cd..8b011a928a 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -151,7 +151,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, @@ -2001,7 +2002,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, @@ -2927,7 +2929,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, @@ -3408,7 +3411,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, @@ -3932,7 +3936,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, @@ -4380,7 +4385,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, @@ -4855,7 +4861,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, @@ -5460,7 +5467,8 @@ If webhook is set to have Event Grid message format then the payload will look a "unique_inputs", "unique_reports", "regression_reports", - "logs" + "logs", + "extra" ], "title": "ContainerType" }, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index 6bf9bea187..76381fb9c2 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -98,7 +98,8 @@ public enum ContainerType { UniqueInputs, UniqueReports, RegressionReports, - Logs + Logs, + Extra } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 494b9378c5..17f4c6c6fa 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -893,6 +893,7 @@ public record TaskDefinition( public record WorkSet( bool Reboot, Uri SetupUrl, + Uri? ExtraUrl, bool Script, List WorkUnits ); @@ -1020,6 +1021,7 @@ Uri HeartbeatQueue public IContainerDef? UniqueInputs { get; set; } public IContainerDef? UniqueReports { get; set; } public IContainerDef? RegressionReports { get; set; } + public IContainerDef? Extra { get; set; } } diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs index a845195b6d..5cec286473 100644 --- a/src/ApiService/ApiService/onefuzzlib/Config.cs +++ b/src/ApiService/ApiService/onefuzzlib/Config.cs @@ -140,6 +140,9 @@ await _containers.GetContainerSasUrl(x.Item2.Name, StorageType.Corpus, ConvertPe case ContainerType.RegressionReports: config.RegressionReports = def; break; + case ContainerType.Extra: + config.Extra = def; + break; } } diff --git a/src/ApiService/ApiService/onefuzzlib/Defs.cs b/src/ApiService/ApiService/onefuzzlib/Defs.cs index 1a53aa2a71..aacd347075 100644 --- a/src/ApiService/ApiService/onefuzzlib/Defs.cs +++ b/src/ApiService/ApiService/onefuzzlib/Defs.cs @@ -40,7 +40,14 @@ public static class Defs { ContainerPermission.List | ContainerPermission.Read | ContainerPermission.Write - )}, + ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + }, MonitorQueue: ContainerType.ReadonlyInputs) }, { @@ -83,6 +90,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), }, MonitorQueue: ContainerType.ReadonlyInputs) }, @@ -135,6 +148,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), }, MonitorQueue: ContainerType.Crashes ) @@ -185,6 +204,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), } )}, { TaskType.GenericAnalysis , @@ -222,6 +247,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), }, MonitorQueue: ContainerType.Crashes) }, @@ -264,6 +295,12 @@ public static class Defs { Value: 0, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), } )}, { @@ -309,6 +346,12 @@ public static class Defs { Value: 1, Permissions: ContainerPermission.Write ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), }, MonitorQueue: ContainerType.Crashes ) @@ -343,7 +386,16 @@ public static class Defs { ContainerPermission.List | ContainerPermission.Read | ContainerPermission.Write - )}, + ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + + + }, MonitorQueue: ContainerType.ReadonlyInputs )}, { @@ -377,6 +429,12 @@ public static class Defs { Value: 0, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), } ) }, @@ -445,6 +503,12 @@ public static class Defs { Value: 1, Permissions: ContainerPermission.Write | ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), } ) }, @@ -486,6 +550,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Write| ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), } ) }, @@ -532,6 +602,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), } ) }, @@ -579,6 +655,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Write ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), }, MonitorQueue: ContainerType.Crashes ) @@ -640,6 +722,12 @@ public static class Defs { Value:1, Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), }) }, { @@ -699,6 +787,12 @@ public static class Defs { Permissions: ContainerPermission.Read | ContainerPermission.List ), + new ContainerDefinition( + Type:ContainerType.Extra, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), }) }, }; diff --git a/src/ApiService/ApiService/onefuzzlib/Scheduler.cs b/src/ApiService/ApiService/onefuzzlib/Scheduler.cs index cfce46bc1f..71685c2b46 100644 --- a/src/ApiService/ApiService/onefuzzlib/Scheduler.cs +++ b/src/ApiService/ApiService/onefuzzlib/Scheduler.cs @@ -104,10 +104,12 @@ private async Async.Task ScheduleWorkset(WorkSet workSet, Pool pool, long if (bucketConfig is not null) { var setupUrl = await _containers.GetContainerSasUrl(bucketConfig.setupContainer, StorageType.Corpus, BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List); + var extraUrl = bucketConfig.extraContainer != null ? await _containers.GetContainerSasUrl(bucketConfig.extraContainer, StorageType.Corpus, BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List) : null; var workSet = new WorkSet( Reboot: bucketConfig.reboot, Script: bucketConfig.setupScript is not null, SetupUrl: setupUrl, + ExtraUrl: extraUrl, WorkUnits: workUnits ); @@ -118,7 +120,7 @@ private async Async.Task ScheduleWorkset(WorkSet workSet, Pool pool, long } - sealed record BucketConfig(long count, bool reboot, Container setupContainer, string? setupScript, Pool pool); + sealed record BucketConfig(long count, bool reboot, Container setupContainer, Container? extraContainer, string? setupScript, Pool pool); sealed record PoolKey( PoolName? poolName = null, @@ -172,6 +174,8 @@ sealed record PoolKey( } var setupContainer = task.Config.Containers?.FirstOrDefault(c => c.Type == ContainerType.Setup) ?? throw new Exception($"task missing setup container: task_type = {task.Config.Task.Type}"); + var extraContainer = task.Config.Containers?.FirstOrDefault(c => c.Type == ContainerType.Extra); + string? setupScript = null; if (task.Os == Os.Windows) { if (await _containers.BlobExists(setupContainer.Name, "setup.ps1", StorageType.Corpus)) { @@ -209,6 +213,7 @@ sealed record PoolKey( count, reboot, setupContainer.Name, + extraContainer?.Name, setupScript, pool with { ETag = default, TimeStamp = default }); diff --git a/src/agent/coverage/src/binary.rs b/src/agent/coverage/src/binary.rs index 2360df768a..6fdd0c308e 100644 --- a/src/agent/coverage/src/binary.rs +++ b/src/agent/coverage/src/binary.rs @@ -129,7 +129,7 @@ pub fn find_coverage_sites( // Apply allowlists per block, to account for inlining. The `location` values // here describe the top of the inline-inclusive call stack. - if !allowlist.source_files.is_allowed(&path) { + if !allowlist.source_files.is_allowed(path) { continue; } diff --git a/src/agent/onefuzz-agent/src/agent/tests.rs b/src/agent/onefuzz-agent/src/agent/tests.rs index f94c40f6fe..1efade1cad 100644 --- a/src/agent/onefuzz-agent/src/agent/tests.rs +++ b/src/agent/onefuzz-agent/src/agent/tests.rs @@ -60,6 +60,7 @@ impl Fixture { WorkSet { reboot: false, setup_url: self.setup_url(), + extra_url: None, script: false, work_units: vec![self.work_unit()], } diff --git a/src/agent/onefuzz-agent/src/debug.rs b/src/agent/onefuzz-agent/src/debug.rs index eccfae4473..ff75f0746a 100644 --- a/src/agent/onefuzz-agent/src/debug.rs +++ b/src/agent/onefuzz-agent/src/debug.rs @@ -143,6 +143,9 @@ pub struct RunWorkerOpt { #[clap(long)] script: bool, + + #[clap(long)] + extra_url: Option, } fn debug_run_worker(opt: RunWorkerOpt) -> Result<()> { @@ -164,6 +167,7 @@ fn debug_run_worker(opt: RunWorkerOpt) -> Result<()> { let work_set = WorkSet { reboot: false, setup_url: BlobContainerUrl::new(opt.setup_url)?, + extra_url: opt.extra_url.map(BlobContainerUrl::new).transpose()?, script: opt.script, work_units: vec![work_unit], }; @@ -188,8 +192,9 @@ async fn run_worker(mut work_set: WorkSet) -> Result> { let mut events = vec![]; let work_unit = work_set.work_units.pop().unwrap(); let setup_dir = work_set.setup_dir()?; + let extra_dir = work_set.extra_dir()?; - let mut worker = Worker::new(&setup_dir, work_unit); + let mut worker = Worker::new(&setup_dir, extra_dir, work_unit); while !worker.is_done() { worker = worker .update( diff --git a/src/agent/onefuzz-agent/src/scheduler.rs b/src/agent/onefuzz-agent/src/scheduler.rs index 4179dddad2..11ba2efcc8 100644 --- a/src/agent/onefuzz-agent/src/scheduler.rs +++ b/src/agent/onefuzz-agent/src/scheduler.rs @@ -250,8 +250,9 @@ impl State { pub async fn run(self) -> Result> { let mut workers = vec![]; let setup_dir = &self.ctx.work_set.setup_dir()?; + let extra_dir = self.ctx.work_set.extra_dir()?; for work in self.ctx.work_set.work_units { - let worker = Some(Worker::new(setup_dir, work)); + let worker = Some(Worker::new(setup_dir, extra_dir.clone(), work)); workers.push(worker); } diff --git a/src/agent/onefuzz-agent/src/setup.rs b/src/agent/onefuzz-agent/src/setup.rs index a1caac3abc..f5834c348b 100644 --- a/src/agent/onefuzz-agent/src/setup.rs +++ b/src/agent/onefuzz-agent/src/setup.rs @@ -43,6 +43,20 @@ pub struct SetupRunner { impl SetupRunner { pub async fn run(&self, work_set: &WorkSet) -> Result { + if let (Some(extra_url), Some(extra_dir)) = (&work_set.extra_url, work_set.extra_dir()?) { + info!("downloading extra container"); + // `azcopy sync` requires the local dir to exist. + fs::create_dir_all(&extra_dir).await.with_context(|| { + format!("unable to create extra container: {}", extra_dir.display()) + })?; + az_copy::sync(extra_url.to_string(), &extra_dir, false).await?; + debug!( + "synced extra container from {} to {}", + extra_url, + extra_dir.display(), + ); + } + info!("running setup for work set"); work_set.save_context(self.machine_id).await?; // Download the setup container. diff --git a/src/agent/onefuzz-agent/src/work.rs b/src/agent/onefuzz-agent/src/work.rs index 0fd746b988..c68aabc493 100644 --- a/src/agent/onefuzz-agent/src/work.rs +++ b/src/agent/onefuzz-agent/src/work.rs @@ -22,6 +22,7 @@ pub type TaskId = Uuid; pub struct WorkSet { pub reboot: bool, pub setup_url: BlobContainerUrl, + pub extra_url: Option, pub script: bool, pub work_units: Vec, } @@ -92,6 +93,21 @@ impl WorkSet { .join("blob-containers") .join(setup_dir)) } + + pub fn extra_dir(&self) -> Result> { + if let Some(extra_url) = &self.extra_url { + let extra_dir = extra_url + .account() + .ok_or_else(|| anyhow!("Invalid container Url"))?; + Ok(Some( + onefuzz::fs::onefuzz_root()? + .join("blob-containers") + .join(extra_dir), + )) + } else { + Ok(None) + } + } } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] diff --git a/src/agent/onefuzz-agent/src/worker.rs b/src/agent/onefuzz-agent/src/worker.rs index 3e12523a6e..295062b3ab 100644 --- a/src/agent/onefuzz-agent/src/worker.rs +++ b/src/agent/onefuzz-agent/src/worker.rs @@ -45,9 +45,14 @@ pub enum Worker { } impl Worker { - pub fn new(setup_dir: impl AsRef, work: WorkUnit) -> Self { + pub fn new( + setup_dir: impl AsRef, + extra_dir: Option>, + work: WorkUnit, + ) -> Self { let ctx = Ready { setup_dir: PathBuf::from(setup_dir.as_ref()), + extra_dir: extra_dir.map(|dir| PathBuf::from(dir.as_ref())), }; let state = State { ctx, work }; state.into() @@ -98,6 +103,7 @@ impl Worker { #[derive(Debug)] pub struct Ready { setup_dir: PathBuf, + extra_dir: Option, } #[derive(Debug)] @@ -130,7 +136,9 @@ impl State { impl State { pub async fn run(self, runner: &mut dyn IWorkerRunner) -> Result> { - let child = runner.run(&self.ctx.setup_dir, &self.work).await?; + let child = runner + .run(&self.ctx.setup_dir, self.ctx.extra_dir, &self.work) + .await?; let state = State { ctx: Running { child }, @@ -189,7 +197,12 @@ impl_from_state_for_worker!(Done); #[async_trait] pub trait IWorkerRunner: Downcast { - async fn run(&self, setup_dir: &Path, work: &WorkUnit) -> Result>; + async fn run( + &self, + setup_dir: &Path, + extra_dir: Option, + work: &WorkUnit, + ) -> Result>; } impl_downcast!(IWorkerRunner); @@ -214,7 +227,12 @@ impl WorkerRunner { #[async_trait] impl IWorkerRunner for WorkerRunner { - async fn run(&self, setup_dir: &Path, work: &WorkUnit) -> Result> { + async fn run( + &self, + setup_dir: &Path, + extra_dir: Option, + work: &WorkUnit, + ) -> Result> { let working_dir = work.working_dir(self.machine_identity.machine_id)?; debug!("worker working dir = {}", working_dir.display()); @@ -260,6 +278,11 @@ impl IWorkerRunner for WorkerRunner { cmd.arg("managed"); cmd.arg("config.json"); cmd.arg(setup_dir); + if let Some(extra_dir) = extra_dir { + cmd.arg("--extra_dir"); + cmd.arg(extra_dir); + } + cmd.stderr(Stdio::piped()); cmd.stdout(Stdio::piped()); diff --git a/src/agent/onefuzz-agent/src/worker/double.rs b/src/agent/onefuzz-agent/src/worker/double.rs index fb64555a76..e46a265b39 100644 --- a/src/agent/onefuzz-agent/src/worker/double.rs +++ b/src/agent/onefuzz-agent/src/worker/double.rs @@ -10,7 +10,12 @@ pub struct WorkerRunnerDouble { #[async_trait] impl IWorkerRunner for WorkerRunnerDouble { - async fn run(&self, _setup_dir: &Path, _work: &WorkUnit) -> Result> { + async fn run( + &self, + _setup_dir: &Path, + _extra_dir: Option, + _work: &WorkUnit, + ) -> Result> { Ok(Box::new(self.child.clone())) } } diff --git a/src/agent/onefuzz-agent/src/worker/tests.rs b/src/agent/onefuzz-agent/src/worker/tests.rs index 20f857858c..62fad0b1aa 100644 --- a/src/agent/onefuzz-agent/src/worker/tests.rs +++ b/src/agent/onefuzz-agent/src/worker/tests.rs @@ -55,7 +55,12 @@ struct RunnerDouble { #[async_trait] impl IWorkerRunner for RunnerDouble { - async fn run(&self, _setup_dir: &Path, _work: &WorkUnit) -> Result> { + async fn run( + &self, + _setup_dir: &Path, + _extra_dir: Option, + _work: &WorkUnit, + ) -> Result> { Ok(Box::new(self.child.clone())) } } @@ -66,6 +71,7 @@ async fn test_ready_run() { let state = State { ctx: Ready { setup_dir: PathBuf::default(), + extra_dir: None, }, work: Fixture.work(), }; @@ -148,6 +154,7 @@ async fn test_worker_ready_update() { let state = State { ctx: Ready { setup_dir: PathBuf::default(), + extra_dir: None, }, work: Fixture.work(), }; diff --git a/src/agent/onefuzz-task/src/local/common.rs b/src/agent/onefuzz-task/src/local/common.rs index 73118d24ee..d2a93e10d4 100644 --- a/src/agent/onefuzz-task/src/local/common.rs +++ b/src/agent/onefuzz-task/src/local/common.rs @@ -22,6 +22,7 @@ use crate::tasks::config::CommonConfig; use crate::tasks::utils::parse_key_value; pub const SETUP_DIR: &str = "setup_dir"; +pub const EXTRA_DIR: &str = "extra_dir"; pub const INPUTS_DIR: &str = "inputs_dir"; pub const CRASHES_DIR: &str = "crashes_dir"; pub const TARGET_WORKERS: &str = "target_workers"; @@ -237,6 +238,7 @@ pub async fn build_local_context( }); let instance_id = get_uuid("instance_id", args).unwrap_or_default(); + let extra_dir = args.get_one::(EXTRA_DIR).cloned(); let setup_dir = if let Some(setup_dir) = args.get_one::(SETUP_DIR) { setup_dir.clone() } else if let Some(target_exe) = args.get_one::(TARGET_EXE) { @@ -253,6 +255,7 @@ pub async fn build_local_context( task_id, instance_id, setup_dir, + extra_dir, machine_identity: MachineIdentity { machine_id: Uuid::nil(), machine_name: "local".to_string(), diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs index af5468d4e6..78535151fa 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_test_input.rs @@ -29,6 +29,7 @@ pub async fn run(args: &clap::ArgMatches, event_sender: Option>) .get_one::(CHECK_RETRY_COUNT) .copied() .expect("has a default value"); + let extra_dir = context.common_config.extra_dir.as_deref(); let config = TestInputArgs { target_exe: target_exe.as_path(), @@ -41,6 +42,7 @@ pub async fn run(args: &clap::ArgMatches, event_sender: Option>) target_timeout, check_retry_count, setup_dir: &context.common_config.setup_dir, + extra_dir, minimized_stack_depth: None, machine_identity: context.common_config.machine_identity, }; diff --git a/src/agent/onefuzz-task/src/local/test_input.rs b/src/agent/onefuzz-task/src/local/test_input.rs index 42076b3f27..53da81c63a 100644 --- a/src/agent/onefuzz-task/src/local/test_input.rs +++ b/src/agent/onefuzz-task/src/local/test_input.rs @@ -44,6 +44,7 @@ pub async fn run(args: &clap::ArgMatches, event_sender: Option>) target_timeout, check_retry_count, setup_dir: &context.common_config.setup_dir, + extra_dir: context.common_config.extra_dir.as_deref(), minimized_stack_depth: None, check_asan_log, check_debugger, diff --git a/src/agent/onefuzz-task/src/managed/cmd.rs b/src/agent/onefuzz-task/src/managed/cmd.rs index 9e80ed7a0d..d746d5af8a 100644 --- a/src/agent/onefuzz-task/src/managed/cmd.rs +++ b/src/agent/onefuzz-task/src/managed/cmd.rs @@ -24,7 +24,8 @@ pub async fn run(args: &clap::ArgMatches) -> Result<()> { .get_one::("setup_dir") .expect("marked as required"); - let config = Config::from_file(config_path, setup_dir)?; + let extra_dir = args.get_one::("extra_dir").map(|f| f.as_path()); + let config = Config::from_file(config_path, setup_dir, extra_dir)?; init_telemetry(config.common()).await; @@ -138,4 +139,9 @@ pub fn args(name: &'static str) -> Command { .required(true) .value_parser(value_parser!(PathBuf)), ) + .arg( + Arg::new("extra_dir") + .required(false) + .value_parser(value_parser!(PathBuf)), + ) } diff --git a/src/agent/onefuzz-task/src/tasks/analysis/generic.rs b/src/agent/onefuzz-task/src/tasks/analysis/generic.rs index 80dc852a34..50249e6bea 100644 --- a/src/agent/onefuzz-task/src/tasks/analysis/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/analysis/generic.rs @@ -209,6 +209,9 @@ pub async fn run_tool( .output_dir(&config.analysis.local_path) .tools_dir(&config.tools.local_path) .setup_dir(&config.common.setup_dir) + .set_optional_ref(&config.common.extra_dir, |expand, extra_dir| { + expand.extra_dir(extra_dir) + }) .job_id(&config.common.job_id) .task_id(&config.common.task_id) .set_optional_ref(&config.common.microsoft_telemetry_key, |tester, key| { diff --git a/src/agent/onefuzz-task/src/tasks/config.rs b/src/agent/onefuzz-task/src/tasks/config.rs index 585d23d7c1..12dbeea6b2 100644 --- a/src/agent/onefuzz-task/src/tasks/config.rs +++ b/src/agent/onefuzz-task/src/tasks/config.rs @@ -55,6 +55,9 @@ pub struct CommonConfig { #[serde(default)] pub setup_dir: PathBuf, + #[serde(default)] + pub extra_dir: Option, + /// Lower bound on available system memory. If the available memory drops /// below the limit, the task will exit with an error. This is a fail-fast /// mechanism to support debugging. @@ -139,13 +142,15 @@ pub enum Config { } impl Config { - pub fn from_file(path: &Path, setup_dir: &Path) -> Result { + pub fn from_file(path: &Path, setup_dir: &Path, extra_dir: Option<&Path>) -> Result { let json = std::fs::read_to_string(path)?; let json_config: serde_json::Value = serde_json::from_str(&json)?; // override the setup_dir in the config file with the parameter value if specified let mut config: Self = serde_json::from_value(json_config)?; + config.common_mut().setup_dir = setup_dir.to_owned(); + config.common_mut().extra_dir = extra_dir.map(|x| x.to_owned()); Ok(config) } diff --git a/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs b/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs index b2c1dfbf94..1822795426 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs @@ -299,6 +299,9 @@ impl<'a> TaskContext<'a> { .input_path(input) .job_id(&self.config.common.job_id) .setup_dir(&self.config.common.setup_dir) + .set_optional_ref(&self.config.common.extra_dir, |expand, extra_dir| { + expand.extra_dir(extra_dir) + }) .target_exe(&target_exe) .target_options(&self.config.target_options) .task_id(&self.config.common.task_id); diff --git a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs index 07022baa4a..5b50ed38cb 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs @@ -299,6 +299,9 @@ impl<'a> TaskContext<'a> { .input_path(input) .job_id(&self.config.common.job_id) .setup_dir(&self.config.common.setup_dir) + .set_optional_ref(&self.config.common.extra_dir, |expand, extra_dir| { + expand.extra_dir(extra_dir) + }) .target_exe(&target_exe) .target_options(&self.config.target_options) .task_id(&self.config.common.task_id); diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs index 448b716a0c..1879306ee1 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/generator.rs @@ -99,6 +99,7 @@ impl GeneratorTask { let tester = Tester::new( &self.config.common.setup_dir, + self.config.common.extra_dir.as_deref(), &target_exe, &self.config.target_options, &self.config.target_env, @@ -168,6 +169,9 @@ impl GeneratorTask { .machine_id() .await? .setup_dir(&self.config.common.setup_dir) + .set_optional_ref(&self.config.common.extra_dir, |expand, extra_dir| { + expand.extra_dir(extra_dir) + }) .generated_inputs(&output_dir) .input_corpus(&corpus_dir) .generator_exe(&self.config.generator_exe) @@ -298,6 +302,7 @@ mod tests { microsoft_telemetry_key: Default::default(), logs: Default::default(), setup_dir: Default::default(), + extra_dir: Default::default(), min_available_memory_mb: Default::default(), machine_identity: onefuzz::machine_id::MachineIdentity { machine_id: uuid::Uuid::new_v4(), diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs index 9bf36eb5a0..791a86e3be 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs @@ -86,6 +86,7 @@ impl common::LibFuzzerType for LibFuzzerDotnet { options, env, &config.common.setup_dir, + config.common.extra_dir.as_ref(), config.common.machine_identity.clone(), )) } diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs index dc870523c9..12fced134f 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs @@ -28,6 +28,7 @@ impl common::LibFuzzerType for GenericLibFuzzer { config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.extra_dir.as_ref(), config.common.machine_identity.clone(), )) } diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs index 1b64fe9a86..2cdf751190 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs @@ -214,6 +214,9 @@ async fn start_supervisor( .input_corpus(&inputs.local_path) .reports_dir(reports_dir) .setup_dir(&config.common.setup_dir) + .set_optional_ref(&config.common.extra_dir, |expand, extra_dir| { + expand.extra_dir(extra_dir) + }) .job_id(&config.common.job_id) .task_id(&config.common.task_id) .set_optional_ref(&config.tools, |expand, tools| { @@ -391,6 +394,7 @@ mod tests { microsoft_telemetry_key: Default::default(), logs: Default::default(), setup_dir: Default::default(), + extra_dir: Default::default(), min_available_memory_mb: Default::default(), machine_identity: MachineIdentity { machine_id: uuid::Uuid::new_v4(), diff --git a/src/agent/onefuzz-task/src/tasks/merge/generic.rs b/src/agent/onefuzz-task/src/tasks/merge/generic.rs index de804154cf..27ea4abd40 100644 --- a/src/agent/onefuzz-task/src/tasks/merge/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/merge/generic.rs @@ -141,6 +141,9 @@ async fn merge(config: &Config, output_dir: impl AsRef) -> Result<()> { .generated_inputs(output_dir) .target_exe(&target_exe) .setup_dir(&config.common.setup_dir) + .set_optional_ref(&config.common.extra_dir, |expand, extra_dir| { + expand.extra_dir(extra_dir) + }) .tools_dir(&config.tools.local_path) .job_id(&config.common.job_id) .task_id(&config.common.task_id) diff --git a/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs b/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs index 7b8c6f7b43..fa4257f4a0 100644 --- a/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs +++ b/src/agent/onefuzz-task/src/tasks/merge/libfuzzer_merge.rs @@ -46,6 +46,7 @@ pub async fn spawn(config: Arc) -> Result<()> { config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.extra_dir.clone(), config.common.machine_identity.clone(), ); fuzzer.verify(config.check_fuzzer_help, None).await?; @@ -160,6 +161,7 @@ pub async fn merge_inputs( config.target_options.clone(), config.target_env.clone(), &config.common.setup_dir, + config.common.extra_dir.clone(), config.common.machine_identity.clone(), ); merger diff --git a/src/agent/onefuzz-task/src/tasks/regression/generic.rs b/src/agent/onefuzz-task/src/tasks/regression/generic.rs index 68d0f1643c..c7be907c70 100644 --- a/src/agent/onefuzz-task/src/tasks/regression/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/regression/generic.rs @@ -60,6 +60,7 @@ impl RegressionHandler for GenericRegressionTask { try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe) .await?; + let extra_dir = self.config.common.extra_dir.as_deref(); let args = generic::TestInputArgs { input_url: Some(input_url), input: &input, @@ -67,6 +68,7 @@ impl RegressionHandler for GenericRegressionTask { target_options: &self.config.target_options, target_env: &self.config.target_env, setup_dir: &self.config.common.setup_dir, + extra_dir, task_id: self.config.common.task_id, job_id: self.config.common.job_id, target_timeout: self.config.target_timeout, diff --git a/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs b/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs index 5a5ce3990b..fbeeaace06 100644 --- a/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs +++ b/src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs @@ -66,6 +66,7 @@ impl RegressionHandler for LibFuzzerRegressionTask { target_options: &self.config.target_options, target_env: &self.config.target_env, setup_dir: &self.config.common.setup_dir, + extra_dir: self.config.common.extra_dir.as_deref(), task_id: self.config.common.task_id, job_id: self.config.common.job_id, target_timeout: self.config.target_timeout, diff --git a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs index 4c8c04325d..8e6ae6650f 100644 --- a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs @@ -182,7 +182,10 @@ impl AsanProcessor { let expand = Expand::new(&self.config.common.machine_identity) .input_path(input) - .setup_dir(&self.config.common.setup_dir); + .setup_dir(&self.config.common.setup_dir) + .set_optional_ref(&self.config.common.extra_dir, |expand, extra_dir| { + expand.extra_dir(extra_dir) + }); let expanded_args = expand.evaluate(&args)?; let env = { diff --git a/src/agent/onefuzz-task/src/tasks/report/generic.rs b/src/agent/onefuzz-task/src/tasks/report/generic.rs index 7b5d2f265c..a02a5be632 100644 --- a/src/agent/onefuzz-task/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/report/generic.rs @@ -113,6 +113,7 @@ pub struct TestInputArgs<'a> { pub target_options: &'a [String], pub target_env: &'a HashMap, pub setup_dir: &'a Path, + pub extra_dir: Option<&'a Path>, pub task_id: Uuid, pub job_id: Uuid, pub target_timeout: Option, @@ -124,8 +125,10 @@ pub struct TestInputArgs<'a> { } pub async fn test_input(args: TestInputArgs<'_>) -> Result { + let extra_dir = args.extra_dir; let tester = Tester::new( args.setup_dir, + extra_dir, args.target_exe, args.target_options, args.target_env, @@ -201,6 +204,7 @@ impl<'a> GenericReportProcessor<'a> { try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe) .await?; + let extra_dir = self.config.common.extra_dir.as_deref(); let args = TestInputArgs { input_url, input, @@ -208,6 +212,7 @@ impl<'a> GenericReportProcessor<'a> { target_options: &self.config.target_options, target_env: &self.config.target_env, setup_dir: &self.config.common.setup_dir, + extra_dir, task_id: self.config.common.task_id, job_id: self.config.common.job_id, target_timeout: self.config.target_timeout, diff --git a/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs index e0d31685d7..87f4f5583b 100644 --- a/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs @@ -76,6 +76,7 @@ impl ReportTask { self.config.target_options.clone(), self.config.target_env.clone(), &self.config.common.setup_dir, + self.config.common.extra_dir.clone(), self.config.common.machine_identity.clone(), ); fuzzer.verify(self.config.check_fuzzer_help, None).await @@ -118,6 +119,7 @@ pub struct TestInputArgs<'a> { pub target_options: &'a [String], pub target_env: &'a HashMap, pub setup_dir: &'a Path, + pub extra_dir: Option<&'a Path>, pub task_id: uuid::Uuid, pub job_id: uuid::Uuid, pub target_timeout: Option, @@ -132,6 +134,7 @@ pub async fn test_input(args: TestInputArgs<'_>) -> Result { args.target_options.to_vec(), args.target_env.clone(), args.setup_dir, + args.extra_dir.map(PathBuf::from), args.machine_identity, ); @@ -215,6 +218,7 @@ impl AsanProcessor { target_options: &self.config.target_options, target_env: &self.config.target_env, setup_dir: &self.config.common.setup_dir, + extra_dir: self.config.common.extra_dir.as_deref(), task_id: self.config.common.task_id, job_id: self.config.common.job_id, target_timeout: self.config.target_timeout, diff --git a/src/agent/onefuzz/examples/test-input.rs b/src/agent/onefuzz/examples/test-input.rs index d834b5f854..6fc844ef69 100644 --- a/src/agent/onefuzz/examples/test-input.rs +++ b/src/agent/onefuzz/examples/test-input.rs @@ -55,6 +55,7 @@ async fn main() -> Result<()> { let env = Default::default(); let tester = Tester::new( &setup_dir, + None, &opt.exe, &target_options, &env, diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index a4688281e2..c7a5f3cd7e 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -41,6 +41,7 @@ pub enum PlaceHolder { SupervisorExe, SupervisorOptions, SetupDir, + ExtraDir, ReportsDir, JobId, TaskId, @@ -74,6 +75,7 @@ impl PlaceHolder { Self::SupervisorExe => "{supervisor_exe}", Self::SupervisorOptions => "{supervisor_options}", Self::SetupDir => "{setup_dir}", + Self::ExtraDir => "{extra_dir}", Self::ReportsDir => "{reports_dir}", Self::JobId => "{job_id}", Self::TaskId => "{task_id}", @@ -317,6 +319,12 @@ impl<'a> Expand<'a> { self.set_value(PlaceHolder::SetupDir, ExpandedValue::Path(path)) } + pub fn extra_dir(self, arg: impl AsRef) -> Self { + let arg = arg.as_ref(); + let path = String::from(arg.to_string_lossy()); + self.set_value(PlaceHolder::ExtraDir, ExpandedValue::Path(path)) + } + pub fn coverage_dir(self, arg: impl AsRef) -> Self { let arg = arg.as_ref(); let path = String::from(arg.to_string_lossy()); diff --git a/src/agent/onefuzz/src/input_tester.rs b/src/agent/onefuzz/src/input_tester.rs index e701273565..09a190e08a 100644 --- a/src/agent/onefuzz/src/input_tester.rs +++ b/src/agent/onefuzz/src/input_tester.rs @@ -27,6 +27,7 @@ const CRASH_SITE_UNAVAILABLE: &str = ""; pub struct Tester<'a> { setup_dir: &'a Path, + extra_dir: Option<&'a Path>, exe_path: &'a Path, arguments: &'a [String], environ: &'a HashMap, @@ -56,6 +57,7 @@ pub struct TestResult { impl<'a> Tester<'a> { pub fn new( setup_dir: &'a Path, + extra_dir: Option<&'a Path>, exe_path: &'a Path, arguments: &'a [String], environ: &'a HashMap, @@ -63,6 +65,7 @@ impl<'a> Tester<'a> { ) -> Self { Self { setup_dir, + extra_dir, exe_path, arguments, environ, @@ -298,7 +301,10 @@ impl<'a> Tester<'a> { .input_path(input_file) .target_exe(self.exe_path) .target_options(self.arguments) - .setup_dir(self.setup_dir); + .setup_dir(self.setup_dir) + .set_optional(self.extra_dir.as_ref(), |expand, extra_dir| { + expand.extra_dir(extra_dir) + }); let argv = expand.evaluate(self.arguments)?; let mut env: HashMap = HashMap::new(); diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index 624cc0aab8..ef13bfa7bf 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -37,6 +37,7 @@ pub struct LibFuzzerMergeOutput { pub struct LibFuzzer { setup_dir: PathBuf, + extra_dir: Option, exe: PathBuf, options: Vec, env: HashMap, @@ -49,6 +50,7 @@ impl LibFuzzer { options: Vec, env: HashMap, setup_dir: impl Into, + extra_dir: Option>, machine_identity: MachineIdentity, ) -> Self { Self { @@ -56,6 +58,7 @@ impl LibFuzzer { options, env, setup_dir: setup_dir.into(), + extra_dir: extra_dir.map(|x| x.into()), machine_identity, } } @@ -108,6 +111,9 @@ impl LibFuzzer { .target_exe(&self.exe) .target_options(&self.options) .setup_dir(&self.setup_dir) + .set_optional(self.extra_dir.as_ref(), |expand, extra_dir| { + expand.extra_dir(extra_dir) + }) .set_optional(corpus_dir, |e, corpus_dir| e.input_corpus(corpus_dir)) .set_optional(fault_dir, |e, fault_dir| e.crashes(fault_dir)); @@ -314,6 +320,7 @@ impl LibFuzzer { let mut tester = Tester::new( &self.setup_dir, + self.extra_dir.as_deref(), &self.exe, &options, &self.env, @@ -443,6 +450,7 @@ mod tests { options.clone(), env.clone(), temp_setup_dir.path(), + Option::::None, MachineIdentity { machine_id: uuid::Uuid::new_v4(), machine_name: "test-input".into(), @@ -476,6 +484,7 @@ mod tests { options.clone(), env.clone(), temp_setup_dir.path(), + Option::::None, MachineIdentity { machine_id: uuid::Uuid::new_v4(), machine_name: "test-input".into(), diff --git a/src/cli/onefuzz/templates/afl.py b/src/cli/onefuzz/templates/afl.py index 936936078a..f882792f0e 100644 --- a/src/cli/onefuzz/templates/afl.py +++ b/src/cli/onefuzz/templates/afl.py @@ -53,6 +53,7 @@ def basic( notification_config: Optional[NotificationConfig] = None, debug: Optional[List[TaskDebugFlag]] = None, ensemble_sync_delay: Optional[int] = None, + extra_container: Optional[Container] = None, ) -> Optional[Job]: """ Basic AFL job @@ -93,6 +94,7 @@ def basic( ContainerType.reports, ContainerType.unique_reports, ) + if existing_inputs: self.onefuzz.containers.get(existing_inputs) helper.containers[ContainerType.inputs] = existing_inputs @@ -133,6 +135,11 @@ def basic( (ContainerType.inputs, helper.containers[ContainerType.inputs]), ] + if extra_container is not None: + containers.append( + (ContainerType.extra, helper.containers[ContainerType.extra]) + ) + self.logger.info("creating afl fuzz task") fuzzer_task = self.onefuzz.tasks.create( helper.job.job_id, @@ -166,6 +173,11 @@ def basic( ), ] + if extra_container is not None: + report_containers.append( + (ContainerType.extra, helper.containers[ContainerType.extra]) + ) + self.logger.info("creating generic_crash_report task") self.onefuzz.tasks.create( helper.job.job_id, diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index 7e3f83c01b..323ae57a69 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -74,6 +74,7 @@ def _create_tasks( analyzer_options: Optional[List[str]] = None, analyzer_env: Optional[Dict[str, str]] = None, tools: Optional[Container] = None, + extra_container: Optional[Container] = None, ) -> None: target_options = target_options or [] @@ -331,6 +332,7 @@ def basic( analyzer_options: Optional[List[str]] = None, analyzer_env: Optional[Dict[str, str]] = None, tools: Optional[Container] = None, + extra_container: Optional[Container] = None, ) -> Optional[Job]: """ Basic libfuzzer job @@ -413,9 +415,14 @@ def basic( else: source_allowlist_blob_name = None + containers = helper.containers + + if extra_container is not None: + containers[ContainerType.extra] = extra_container + self._create_tasks( job=helper.job, - containers=helper.containers, + containers=containers, pool_name=pool_name, target_exe=target_exe_blob_name, vm_count=vm_count, @@ -474,6 +481,7 @@ def merge( debug: Optional[List[TaskDebugFlag]] = None, preserve_existing_outputs: bool = False, check_fuzzer_help: bool = True, + extra_container: Optional[Container] = None, ) -> Optional[Job]: """ libfuzzer merge task @@ -510,6 +518,7 @@ def merge( helper.define_containers( ContainerType.setup, ) + if inputs: helper.define_containers(ContainerType.inputs) @@ -535,6 +544,9 @@ def merge( ), ] + if extra_container is not None: + merge_containers.append((ContainerType.extra, extra_container)) + if inputs: merge_containers.append( (ContainerType.inputs, helper.containers[ContainerType.inputs]) @@ -598,6 +610,7 @@ def dotnet( colocate_secondary_tasks: bool = True, expect_crash_on_failure: bool = False, notification_config: Optional[NotificationConfig] = None, + extra_container: Optional[Container] = None, ) -> Optional[Job]: pool = self.onefuzz.pools.get(pool_name) @@ -673,6 +686,9 @@ def dotnet( (ContainerType.tools, fuzzer_tools_container), ] + if extra_container is not None: + fuzzer_containers.append((ContainerType.extra, extra_container)) + helper.create_containers() helper.setup_notifications(notification_config) @@ -728,6 +744,9 @@ def dotnet( (ContainerType.tools, fuzzer_tools_container), ] + if extra_container is not None: + coverage_containers.append((ContainerType.extra, extra_container)) + self.logger.info("creating `dotnet_coverage` task") self.onefuzz.tasks.create( helper.job.job_id, @@ -756,6 +775,9 @@ def dotnet( (ContainerType.tools, fuzzer_tools_container), ] + if extra_container is not None: + report_containers.append((ContainerType.extra, extra_container)) + self.logger.info("creating `dotnet_crash_report` task") self.onefuzz.tasks.create( helper.job.job_id, @@ -808,6 +830,7 @@ def qemu_user( crash_report_timeout: Optional[int] = 1, check_retry_count: Optional[int] = 300, check_fuzzer_help: bool = True, + extra_container: Optional[Container] = None, ) -> Optional[Job]: """ libfuzzer tasks, wrapped via qemu-user (PREVIEW FEATURE) @@ -866,6 +889,9 @@ def qemu_user( (ContainerType.inputs, helper.containers[ContainerType.inputs]), ] + if extra_container is not None: + fuzzer_containers.append((ContainerType.extra, extra_container)) + helper.create_containers() target_exe_blob_name = helper.setup_relative_blob_name(target_exe, None) @@ -959,6 +985,9 @@ def qemu_user( (ContainerType.no_repro, helper.containers[ContainerType.no_repro]), ] + if extra_container is not None: + report_containers.append((ContainerType.extra, extra_container)) + self.logger.info("creating libfuzzer_crash_report task") self.onefuzz.tasks.create( helper.job.job_id, diff --git a/src/cli/onefuzz/templates/ossfuzz.py b/src/cli/onefuzz/templates/ossfuzz.py index a135f92e28..7825ccc96d 100644 --- a/src/cli/onefuzz/templates/ossfuzz.py +++ b/src/cli/onefuzz/templates/ossfuzz.py @@ -11,7 +11,7 @@ from onefuzztypes.enums import OS, ContainerType, TaskDebugFlag from onefuzztypes.models import NotificationConfig -from onefuzztypes.primitives import File, PoolName +from onefuzztypes.primitives import Container, File, PoolName from onefuzz.api import Command from onefuzz.backend import container_file_path @@ -119,6 +119,7 @@ def libfuzzer( notification_config: Optional[NotificationConfig] = None, debug: Optional[List[TaskDebugFlag]] = None, ensemble_sync_delay: Optional[int] = None, + extra_container: Optional[Container] = None, ) -> None: """ OssFuzz style libfuzzer jobs @@ -212,6 +213,10 @@ def libfuzzer( ContainerType.no_repro, ContainerType.coverage, ) + + if extra_container is not None: + helper.containers[ContainerType.extra] = extra_container + helper.create_containers() helper.setup_notifications(notification_config) diff --git a/src/cli/onefuzz/templates/radamsa.py b/src/cli/onefuzz/templates/radamsa.py index 64bef0c20e..6ab0d9e20f 100644 --- a/src/cli/onefuzz/templates/radamsa.py +++ b/src/cli/onefuzz/templates/radamsa.py @@ -50,6 +50,7 @@ def basic( debug: Optional[List[TaskDebugFlag]] = None, ensemble_sync_delay: Optional[int] = None, target_timeout: Optional[int] = None, + extra_container: Optional[Container] = None, ) -> Optional[Job]: """ Basic radamsa job @@ -90,6 +91,7 @@ def basic( ContainerType.no_repro, ContainerType.analysis, ) + if existing_inputs: self.onefuzz.containers.get(existing_inputs) helper.containers[ContainerType.readonly_inputs] = existing_inputs @@ -155,6 +157,9 @@ def basic( ), ] + if extra_container is not None: + containers.append((ContainerType.extra, extra_container)) + fuzzer_task = self.onefuzz.tasks.create( helper.job.job_id, TaskType.generic_generator, @@ -188,6 +193,9 @@ def basic( (ContainerType.no_repro, helper.containers[ContainerType.no_repro]), ] + if extra_container is not None: + report_containers.append((ContainerType.extra, extra_container)) + self.logger.info("creating generic_crash_report task") self.onefuzz.tasks.create( helper.job.job_id, @@ -231,6 +239,9 @@ def basic( (ContainerType.crashes, helper.containers[ContainerType.crashes]), ] + if extra_container is not None: + analysis_containers.append((ContainerType.extra, extra_container)) + self.onefuzz.tasks.create( helper.job.job_id, TaskType.generic_analysis, diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index e2ecce7812..10c91a4423 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -56,6 +56,7 @@ def generic( check_fuzzer_help: bool = True, delete_input_container: bool = True, check_regressions: bool = False, + extra_container: Optional[Container] = None, ) -> None: """ generic regression task @@ -89,6 +90,7 @@ def generic( check_fuzzer_help=check_fuzzer_help, delete_input_container=delete_input_container, check_regressions=check_regressions, + extra_container=extra_container, ) def libfuzzer( @@ -115,6 +117,7 @@ def libfuzzer( check_fuzzer_help: bool = True, delete_input_container: bool = True, check_regressions: bool = False, + extra_container: Optional[Container] = None, ) -> None: """ libfuzzer regression task @@ -148,6 +151,7 @@ def libfuzzer( check_fuzzer_help=check_fuzzer_help, delete_input_container=delete_input_container, check_regressions=check_regressions, + extra_container=extra_container, ) def _create_job( @@ -175,6 +179,7 @@ def _create_job( check_fuzzer_help: bool = True, delete_input_container: bool = True, check_regressions: bool = False, + extra_container: Optional[Container] = None, ) -> None: if dryrun: return None @@ -216,6 +221,9 @@ def _create_job( ), ] + if extra_container: + containers.append((ContainerType.extra, extra_container)) + if crashes: helper.containers[ ContainerType.readonly_inputs diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index d14d8007d2..3769c41cb8 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -109,6 +109,7 @@ class Integration(BaseModel): }, reboot_after_setup=True, inject_fake_regression=True, + fuzzing_target_options=["--test:{extra}"], ), "linux-libfuzzer-with-options": Integration( template=TemplateType.libfuzzer, @@ -180,6 +181,7 @@ class Integration(BaseModel): os=OS.linux, target_exe="fuzz_target_1", wait_for_files={ContainerType.unique_reports: 1, ContainerType.coverage: 1}, + fuzzing_target_options=["--test:{extra}"], ), "linux-trivial-crash": Integration( template=TemplateType.radamsa, @@ -209,6 +211,7 @@ class Integration(BaseModel): ContainerType.coverage: 1, }, inject_fake_regression=True, + fuzzing_target_options=["--test:{extra}"], ), "windows-libfuzzer-linked-library": Integration( template=TemplateType.libfuzzer, @@ -575,6 +578,8 @@ def launch( job: Optional[Job] = None if config.template == TemplateType.libfuzzer: + # building the extra container to test this variable substitution + extra = self.of.containers.create("extra") job = self.of.template.libfuzzer.basic( self.project, target, @@ -588,6 +593,7 @@ def launch( reboot_after_setup=config.reboot_after_setup or False, target_options=config.target_options, fuzzing_target_options=config.fuzzing_target_options, + extra_container=Container(extra.name), ) elif config.template == TemplateType.libfuzzer_dotnet: if setup is None: diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index 2d9f4c9c4d..7675093816 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -228,6 +228,7 @@ class ContainerType(Enum): unique_reports = "unique_reports" regression_reports = "regression_reports" logs = "logs" + extra = "extra" @classmethod def reset_defaults(cls) -> List["ContainerType"]: diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 3f5760e1a3..8f2b0e784e 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -408,6 +408,7 @@ class TaskUnitConfig(BaseModel): unique_inputs: CONTAINER_DEF unique_reports: CONTAINER_DEF regression_reports: CONTAINER_DEF + extra: CONTAINER_DEF class Forward(BaseModel):