start work towards enabling launching fuzzing within docker

.github/workflows/ci.yml

@@ -253,8 +253,10 @@ jobs:
           cp artifacts/azcopy/azcopy.exe artifacts/azcopy/ThirdPartyNotice.txt src/deployment/tools/win64
           cp artifacts/agent/onefuzz-supervisor.exe src/deployment/tools/win64/
           cp artifacts/agent/onefuzz-agent.exe src/deployment/tools/win64/
+          cp artifacts/agent/onefuzz-downloader.exe src/deployment/tools/win64/
           cp artifacts/agent/onefuzz-supervisor src/deployment/tools/linux/
           cp artifacts/agent/onefuzz-agent src/deployment/tools/linux/
+          cp artifacts/agent/onefuzz-downloader src/deployment/tools/linux/
           cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/
           cp artifacts/service/api-service.zip src/deployment
           cp -r artifacts/third-party src/deployment

.gitignore

@@ -4,6 +4,6 @@
 .idea
 **/.direnv
 **/.envrc
-
+artifacts
 # vim
 *.swp

src/agent/Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "input-tester",
     "onefuzz",
     "onefuzz-agent",
+    "onefuzz-downloader",
     "onefuzz-supervisor",
     "storage-queue",
     "win-util",

src/agent/onefuzz-downloader/.gitignore

@@ -0,0 +1,3 @@
+/target
+Cargo.lock
+data/licenses.json

src/agent/onefuzz-downloader/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "onefuzz-downloader"
+version = "0.1.0"
+authors = ["fuzzing@microsoft.com"]
+edition = "2018"
+publish = false
+
+
+[dependencies]
+anyhow = "1.0.31"
+appinsights = "0.1"
+async-trait = "0.1.36"
+downcast-rs = "1.2.0"
+env_logger = "0.7"
+futures = "0.3.5"
+log = "0.4"
+onefuzz = { path = "../onefuzz" }
+reqwest = { version = "0.10", features = ["json", "stream"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+storage-queue = { path = "../storage-queue" }
+structopt = "0.3"
+tokio = { version = "0.2.13", features = ["full"] }
+url = { version = "2.1.1", features = ["serde"] }
+uuid = { version = "0.8.1", features = ["serde", "v4"] }
+clap = "2.33"

src/agent/onefuzz-downloader/build.rs

@@ -0,0 +1,56 @@
+use std::env;
+use std::error::Error;
+use std::fs::File;
+use std::io::prelude::*;
+use std::process::Command;
+
+fn run_cmd(args: &[&str]) -> Result<String, Box<dyn Error>> {
+    let cmd = Command::new(args[0]).args(&args[1..]).output()?;
+    if cmd.status.success() {
+        Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string())
+    } else {
+        Err(From::from("failed"))
+    }
+}
+
+fn read_file(filename: &str) -> Result<String, Box<dyn Error>> {
+    let mut file = File::open(filename)?;
+    let mut contents = String::new();
+    file.read_to_string(&mut contents)?;
+    contents = contents.trim().to_string();
+
+    Ok(contents)
+}
+
+fn print_version(include_sha: bool, include_local: bool) -> Result<(), Box<dyn Error>> {
+    let mut version = read_file("../../../CURRENT_VERSION")?;
+    let sha = run_cmd(&["git", "rev-parse", "HEAD"])?;
+
+    if include_sha {
+        version.push('-');
+        version.push_str(&sha);
+
+        // if we're a non-release build, check to see if git has
+        // unstaged changes
+        if include_local && run_cmd(&["git", "diff", "--quiet"]).is_err() {
+            version.push('.');
+            version.push_str("localchanges");
+        }
+    }
+
+    println!("cargo:rustc-env=GIT_VERSION={}", sha);
+    println!("cargo:rustc-env=ONEFUZZ_VERSION={}", version);
+
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    // If we're built off of a tag, we accept CURRENT_VERSION as is.  Otherwise
+    // modify it to indicate local build
+    let (include_sha, include_local_changes) = if let Ok(val) = env::var("GITHUB_REF") {
+        (!val.starts_with("refs/tags/"), false)
+    } else {
+        (true, true)
+    };
+    print_version(include_sha, include_local_changes)
+}

src/agent/onefuzz-downloader/src/config.rs

@@ -0,0 +1,104 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use anyhow::Result;
+use onefuzz::auth::{ClientCredentials, Credentials, ManagedIdentityCredentials};
+use std::path::Path;
+use url::Url;
+use uuid::Uuid;
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
+pub struct StaticConfig {
+    pub credentials: Credentials,
+    pub pool_name: String,
+    pub onefuzz_url: Url,
+    pub instrumentation_key: Option<Uuid>,
+    pub telemetry_key: Option<Uuid>,
+}
+
+// Temporary shim type to bridge the current service-provided config.
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
+struct RawStaticConfig {
+    pub credentials: Option<ClientCredentials>,
+    pub pool_name: String,
+    pub onefuzz_url: Url,
+    pub instrumentation_key: Option<Uuid>,
+    pub telemetry_key: Option<Uuid>,
+}
+
+impl StaticConfig {
+    pub fn new(data: &[u8]) -> Result<Self> {
+        let config: RawStaticConfig = serde_json::from_slice(data)?;
+
+        let credentials = match config.credentials {
+            Some(client) => client.into(),
+            None => {
+                // Remove trailing `/`, which is treated as a distinct resource.
+                let resource = config
+                    .onefuzz_url
+                    .to_string()
+                    .trim_end_matches('/')
+                    .to_owned();
+                let managed = ManagedIdentityCredentials::new(resource);
+                managed.into()
+            }
+        };
+        let config = StaticConfig {
+            credentials,
+            pool_name: config.pool_name,
+            onefuzz_url: config.onefuzz_url,
+            instrumentation_key: config.instrumentation_key,
+            telemetry_key: config.telemetry_key,
+        };
+
+        Ok(config)
+    }
+
+    pub fn from_env() -> Result<Self> {
+        let client_id = Uuid::parse_str(&std::env::var("ONEFUZZ_CLIENT_ID")?)?;
+        let client_secret = std::env::var("ONEFUZZ_CLIENT_SECRET")?.into();
+        let tenant = std::env::var("ONEFUZZ_TENANT")?;
+        let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?;
+        let pool_name = std::env::var("ONEFUZZ_POOL")?;
+
+        let instrumentation_key = if let Ok(key) = std::env::var("ONEFUZZ_INSTRUMENTATION_KEY") {
+            Some(Uuid::parse_str(&key)?)
+        } else {
+            None
+        };
+
+        let telemetry_key = if let Ok(key) = std::env::var("ONEFUZZ_TELEMETRY_KEY") {
+            Some(Uuid::parse_str(&key)?)
+        } else {
+            None
+        };
+
+        let credentials = ClientCredentials::new(
+            client_id,
+            client_secret,
+            onefuzz_url.clone().to_string(),
+            tenant,
+        )
+        .into();
+
+        Ok(Self {
+            credentials,
+            pool_name,
+            onefuzz_url,
+            instrumentation_key,
+            telemetry_key,
+        })
+    }
+
+    pub fn from_file(config_path: impl AsRef<Path>) -> Result<Self> {
+        verbose!("loading config from: {}", config_path.as_ref().display());
+        let data = std::fs::read(config_path)?;
+        Self::new(&data)
+    }
+
+    pub fn download_url(&self) -> Url {
+        let mut url = self.onefuzz_url.clone();
+        url.set_path("/api/agents/download");
+        url
+    }
+}

src/agent/onefuzz-downloader/src/main.rs

@@ -0,0 +1,127 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+#[macro_use]
+extern crate anyhow;
+#[macro_use]
+extern crate onefuzz;
+#[macro_use]
+extern crate serde;
+#[macro_use]
+extern crate clap;
+
+use std::path::PathBuf;
+
+use anyhow::Result;
+use onefuzz::{
+    machine_id::get_machine_id,
+    telemetry::{self, EventData},
+};
+use structopt::StructOpt;
+
+pub mod config;
+pub mod setup;
+
+use config::StaticConfig;
+
+#[derive(StructOpt, Debug)]
+enum Opt {
+    Run(RunOpt),
+    Licenses,
+    Version,
+}
+
+#[derive(StructOpt, Debug)]
+struct RunOpt {
+    #[structopt(short, long = "--config", parse(from_os_str))]
+    config_path: Option<PathBuf>,
+
+    #[structopt(short, long = "--onefuzz_path", parse(from_os_str))]
+    onefuzz_path: Option<PathBuf>,
+
+    #[structopt(short, long)]
+    start_supervisor: bool,
+}
+
+fn main() -> Result<()> {
+    env_logger::init();
+
+    let opt = Opt::from_args();
+
+    match opt {
+        Opt::Run(opt) => run(opt)?,
+        Opt::Licenses => licenses()?,
+        Opt::Version => versions()?,
+    };
+
+    Ok(())
+}
+
+fn versions() -> Result<()> {
+    println!(
+        "{} onefuzz:{} git:{}",
+        crate_version!(),
+        env!("ONEFUZZ_VERSION"),
+        env!("GIT_VERSION")
+    );
+    Ok(())
+}
+
+fn licenses() -> Result<()> {
+    use std::io::{self, Write};
+    io::stdout().write_all(include_bytes!("../../data/licenses.json"))?;
+    Ok(())
+}
+
+fn run(opt: RunOpt) -> Result<()> {
+    // We can't send telemetry if this fails.
+    let config = load_config(&opt);
+
+    if let Err(err) = &config {
+        error!("error loading supervisor agent config: {:?}", err);
+    }
+
+    let config = config?;
+    init_telemetry(&config);
+
+    let mut rt = tokio::runtime::Runtime::new()?;
+    let result = rt.block_on(run_downloader(config, &opt));
+
+    if let Err(err) = &result {
+        error!("error running downloader: {}", err);
+    }
+
+    telemetry::try_flush_and_close();
+
+    result
+}
+
+fn load_config(opt: &RunOpt) -> Result<StaticConfig> {
+    info!("loading downloader config");
+    let config = match &opt.config_path {
+        Some(config_path) => StaticConfig::from_file(config_path)?,
+        None => StaticConfig::from_env()?,
+    };
+
+    Ok(config)
+}
+
+async fn run_downloader(config: StaticConfig, opt: &RunOpt) -> Result<()> {
+    telemetry::set_property(EventData::MachineId(get_machine_id().await?));
+    telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string()));
+
+    let setup_inst = setup::Setup { config };
+    setup_inst.run(&opt.onefuzz_path).await?;
+
+    if opt.start_supervisor {
+        setup_inst.launch_supervisor(&opt.config_path).await?;
+    }
+
+    Ok(())
+}
+
+fn init_telemetry(config: &StaticConfig) {
+    let inst_key = config.instrumentation_key;
+    let tele_key = config.telemetry_key;
+    telemetry::set_appinsights_clients(inst_key, tele_key);
+}