Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Commit

Permalink
unify syncdir management & add jitter (#180)
Browse files Browse the repository at this point in the history
  • Loading branch information
bmc-msft authored Oct 20, 2020
1 parent 041c6ae commit 3dd0f13
Show file tree
Hide file tree
Showing 25 changed files with 273 additions and 280 deletions.
1 change: 1 addition & 0 deletions src/agent/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions src/agent/onefuzz-agent/src/debug/generic_crash_report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// Licensed under the MIT License.

use crate::tasks::{
config::{CommonConfig, SyncedDir},
config::CommonConfig,
report::generic::{Config, GenericReportProcessor},
utils::parse_key_value,
};
use anyhow::Result;
use clap::{App, Arg, SubCommand};
use onefuzz::blob::BlobContainerUrl;
use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir};
use std::{
collections::HashMap,
path::{Path, PathBuf},
Expand Down
4 changes: 2 additions & 2 deletions src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// Licensed under the MIT License.

use crate::tasks::{
config::{CommonConfig, SyncedDir},
config::CommonConfig,
coverage::libfuzzer_coverage::{Config, CoverageProcessor},
utils::parse_key_value,
};
use anyhow::Result;
use clap::{App, Arg, SubCommand};
use onefuzz::blob::BlobContainerUrl;
use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir};
use std::{
collections::HashMap,
path::{Path, PathBuf},
Expand Down
4 changes: 2 additions & 2 deletions src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// Licensed under the MIT License.

use crate::tasks::{
config::{CommonConfig, SyncedDir},
config::CommonConfig,
report::libfuzzer_report::{AsanProcessor, Config},
utils::parse_key_value,
};
use anyhow::Result;
use clap::{App, Arg, SubCommand};
use onefuzz::blob::BlobContainerUrl;
use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir};
use std::{
collections::HashMap,
path::{Path, PathBuf},
Expand Down
4 changes: 2 additions & 2 deletions src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// Licensed under the MIT License.

use crate::tasks::{
config::{CommonConfig, SyncedDir},
config::CommonConfig,
fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask},
utils::parse_key_value,
};
use anyhow::Result;
use clap::{App, Arg, SubCommand};
use onefuzz::blob::BlobContainerUrl;
use onefuzz::{blob::BlobContainerUrl, syncdir::SyncedDir};
use std::{collections::HashMap, path::PathBuf};
use tokio::runtime::Runtime;
use url::Url;
Expand Down
26 changes: 11 additions & 15 deletions src/agent/onefuzz-agent/src/tasks/analysis/generic.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

use crate::tasks::{
config::{CommonConfig, SyncedDir},
heartbeat::HeartbeatSender,
utils,
};
use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender};
use anyhow::Result;
use futures::stream::StreamExt;
use onefuzz::{az_copy, blob::url::BlobUrl};
use onefuzz::{expand::Expand, fs::set_executable, fs::OwnedDir};
use onefuzz::{
expand::Expand, fs::set_executable, fs::OwnedDir, jitter::delay_with_jitter, syncdir::SyncedDir,
};
use reqwest::Url;
use serde::Deserialize;
use std::{
Expand Down Expand Up @@ -43,9 +41,9 @@ pub async fn spawn(config: Config) -> Result<()> {
let tmp = OwnedDir::new(tmp_dir);
tmp.reset().await?;

utils::init_dir(&config.analysis.path).await?;
utils::init_dir(&config.tools.path).await?;
utils::sync_remote_dir(&config.tools, utils::SyncOperation::Pull).await?;
config.analysis.init().await?;
config.tools.init_pull().await?;

set_executable(&config.tools.path).await?;
run_existing(&config).await?;
poll_inputs(&config, tmp).await?;
Expand All @@ -54,16 +52,14 @@ pub async fn spawn(config: Config) -> Result<()> {

async fn run_existing(config: &Config) -> Result<()> {
if let Some(crashes) = &config.crashes {
utils::init_dir(&crashes.path).await?;
utils::sync_remote_dir(&crashes, utils::SyncOperation::Pull).await?;

crashes.init_pull().await?;
let mut read_dir = fs::read_dir(&crashes.path).await?;
while let Some(file) = read_dir.next().await {
verbose!("Processing file {:?}", file);
let file = file?;
run_tool(file.path(), &config).await?;
}
utils::sync_remote_dir(&config.analysis, utils::SyncOperation::Push).await?;
config.analysis.sync_push().await?;
}
Ok(())
}
Expand Down Expand Up @@ -103,12 +99,12 @@ async fn poll_inputs(config: &Config, tmp_dir: OwnedDir) -> Result<()> {
az_copy::copy(input_url.url().as_ref(), &destination_path, false).await?;

run_tool(destination_path, &config).await?;
utils::sync_remote_dir(&config.analysis, utils::SyncOperation::Push).await?;
config.analysis.sync_push().await?
}
input_queue.delete(message).await?;
} else {
warn!("no new candidate inputs found, sleeping");
tokio::time::delay_for(EMPTY_QUEUE_DELAY).await;
delay_with_jitter(EMPTY_QUEUE_DELAY).await;
}
}
}
Expand Down
9 changes: 1 addition & 8 deletions src/agent/onefuzz-agent/src/tasks/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,12 @@
use crate::tasks::{analysis, coverage, fuzz, heartbeat::*, merge, report};
use anyhow::Result;
use onefuzz::{
blob::BlobContainerUrl,
machine_id::{get_machine_id, get_scaleset_name},
telemetry::{self, Event::task_start, EventData},
};
use reqwest::Url;
use serde::{self, Deserialize};
use std::path::{Path, PathBuf};
use std::path::Path;
use std::sync::Arc;
use uuid::Uuid;

Expand Down Expand Up @@ -150,9 +149,3 @@ impl Config {
}
}
}

#[derive(Debug, Deserialize, Clone, PartialEq)]
pub struct SyncedDir {
pub path: PathBuf,
pub url: BlobContainerUrl,
}
29 changes: 7 additions & 22 deletions src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,15 @@
//!
//! Versions in parentheses have been tested.
use crate::tasks::config::SyncedDir;
use crate::tasks::coverage::{recorder::CoverageRecorder, total::TotalCoverage};
use crate::tasks::heartbeat::*;
use crate::tasks::utils::{init_dir, sync_remote_dir, SyncOperation};
use crate::tasks::{config::CommonConfig, generic::input_poller::*};
use anyhow::Result;
use async_trait::async_trait;
use futures::stream::StreamExt;
use onefuzz::fs::list_files;
use onefuzz::telemetry::Event::coverage_data;
use onefuzz::telemetry::EventData;
use onefuzz::{
fs::list_files, syncdir::SyncedDir, telemetry::Event::coverage_data, telemetry::EventData,
};
use reqwest::Url;
use serde::Deserialize;
use std::collections::HashMap;
Expand Down Expand Up @@ -93,19 +91,7 @@ impl CoverageTask {

pub async fn run(&mut self) -> Result<()> {
info!("starting libFuzzer coverage task");

init_dir(&self.config.coverage.path).await?;
verbose!(
"initialized coverage dir, path = {}",
self.config.coverage.path.display()
);

sync_remote_dir(&self.config.coverage, SyncOperation::Pull).await?;
verbose!(
"synced coverage dir, path = {}",
self.config.coverage.path.display()
);

self.config.coverage.init_pull().await?;
self.process().await
}

Expand All @@ -115,12 +101,11 @@ impl CoverageTask {
// Update the total with the coverage from each seed corpus.
for dir in &self.config.readonly_inputs {
verbose!("recording coverage for {}", dir.path.display());
init_dir(&dir.path).await?;
sync_remote_dir(&dir, SyncOperation::Pull).await?;
dir.init_pull().await?;
self.record_corpus_coverage(&mut processor, dir).await?;
fs::remove_dir_all(&dir.path).await?;
sync_remote_dir(&self.config.coverage, SyncOperation::Push).await?;
}
self.config.coverage.sync_push().await?;

info!(
"recorded coverage for {} containers in `readonly_inputs`",
Expand Down Expand Up @@ -246,7 +231,7 @@ impl Processor for CoverageProcessor {
self.heartbeat_client.alive();
self.test_input(input).await?;
self.report_total().await?;
sync_remote_dir(&self.config.coverage, SyncOperation::Push).await?;
self.config.coverage.sync_push().await?;
Ok(())
}
}
44 changes: 15 additions & 29 deletions src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

use crate::tasks::{
config::{CommonConfig, SyncedDir},
heartbeat::*,
utils,
};
use crate::tasks::{config::CommonConfig, heartbeat::*, utils};
use anyhow::{Error, Result};
use futures::stream::StreamExt;
use onefuzz::{
expand::Expand, fs::set_executable, input_tester::Tester, sha256, telemetry::Event::new_result,
expand::Expand,
fs::set_executable,
input_tester::Tester,
sha256,
syncdir::{continuous_sync, SyncOperation::Pull, SyncedDir},
telemetry::Event::new_result,
};
use serde::Deserialize;
use std::collections::HashMap;
Expand Down Expand Up @@ -50,22 +51,18 @@ pub struct GeneratorConfig {
}

pub async fn spawn(config: Arc<GeneratorConfig>) -> Result<(), Error> {
utils::init_dir(&config.crashes.path).await?;
utils::init_dir(&config.tools.path).await?;
utils::sync_remote_dir(&config.tools, utils::SyncOperation::Pull).await?;
config.crashes.init().await?;
config.tools.init_pull().await?;

set_executable(&config.tools.path).await?;
let hb_client = config.common.init_heartbeat().await?;

for sync_dir in &config.readonly_inputs {
utils::init_dir(&sync_dir.path).await?;
utils::sync_remote_dir(&sync_dir, utils::SyncOperation::Pull).await?;
for dir in &config.readonly_inputs {
dir.init_pull().await?;
}

let resync = resync_corpuses(
config.readonly_inputs.clone(),
std::time::Duration::from_secs(10),
);
let crash_dir_monitor = utils::monitor_result_dir(config.crashes.clone(), new_result);
let sync_task = continuous_sync(&config.readonly_inputs, Pull, None);
let crash_dir_monitor = config.crashes.monitor_results(new_result);
let tester = Tester::new(
&config.target_exe,
&config.target_options,
Expand All @@ -78,7 +75,7 @@ pub async fn spawn(config: Arc<GeneratorConfig>) -> Result<(), Error> {
);
let inputs: Vec<_> = config.readonly_inputs.iter().map(|x| &x.path).collect();
let fuzzing_monitor = start_fuzzing(&config, inputs, tester, hb_client);
futures::try_join!(fuzzing_monitor, resync, crash_dir_monitor)?;
futures::try_join!(fuzzing_monitor, sync_task, crash_dir_monitor)?;
Ok(())
}

Expand Down Expand Up @@ -183,17 +180,6 @@ async fn start_fuzzing<'a>(
}
}

pub async fn resync_corpuses(dirs: Vec<SyncedDir>, delay: std::time::Duration) -> Result<()> {
loop {
for sync_dir in &dirs {
utils::sync_remote_dir(sync_dir, utils::SyncOperation::Pull)
.await
.ok();
}
tokio::time::delay_for(delay).await;
}
}

mod tests {
#[tokio::test]
#[cfg(target_os = "linux")]
Expand Down
Loading

0 comments on commit 3dd0f13

Please sign in to comment.