From 68b685f42bd2cd6e49620152f8ec3811e55b37a2 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 11:54:51 -0400 Subject: [PATCH 01/35] start work towards enabling launching fuzzing within docker --- src/ci/azcopy.sh | 21 ++++++++++++--------- src/docker/linux/.gitignore | 1 + src/docker/linux/Dockerfile | 18 ++++++++++++++++++ src/docker/linux/build.sh | 6 ++++++ 4 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 src/docker/linux/.gitignore create mode 100644 src/docker/linux/Dockerfile create mode 100755 src/docker/linux/build.sh diff --git a/src/ci/azcopy.sh b/src/ci/azcopy.sh index 370f6f1b7b..7dafb2980a 100755 --- a/src/ci/azcopy.sh +++ b/src/ci/azcopy.sh @@ -5,12 +5,15 @@ set -ex -mkdir -p artifacts/azcopy - -wget -O azcopy.zip https://aka.ms/downloadazcopy-v10-windows -unzip azcopy.zip -mv azcopy_windows*/* artifacts/azcopy/ - -wget -O azcopy.tgz https://aka.ms/downloadazcopy-v10-linux -tar zxvf azcopy.tgz -mv azcopy_linux_amd64*/* artifacts/azcopy/ +if [ ! -f artifacts/azcopy/azcopy ]; then + mkdir -p artifacts/azcopy + + wget -O azcopy.zip https://aka.ms/downloadazcopy-v10-windows + unzip azcopy.zip + mv azcopy_windows*/* artifacts/azcopy/ + + wget -O azcopy.tgz https://aka.ms/downloadazcopy-v10-linux + tar zxvf azcopy.tgz + mv azcopy_linux_amd64*/* artifacts/azcopy/ + rm -r azcopy* +fi diff --git a/src/docker/linux/.gitignore b/src/docker/linux/.gitignore new file mode 100644 index 0000000000..71e9f312d4 --- /dev/null +++ b/src/docker/linux/.gitignore @@ -0,0 +1 @@ +tools \ No newline at end of file diff --git a/src/docker/linux/Dockerfile b/src/docker/linux/Dockerfile new file mode 100644 index 0000000000..ef3e19f916 --- /dev/null +++ b/src/docker/linux/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:18.04 +LABEL maintainer="fuzzing@microsoft.com" +LABEL ABOUT="OneFuzz fuzzing container" + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y gdb gdbserver llvm-10 wget unzip libssl1.0.0 libunwind-dev \ + && apt-get clean + +RUN mkdir -p /onefuzz/bin +ENV ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer +RUN ln -f -s $(which llvm-symbolizer-10) $ASAN_SYMBOLIZER_PATH + +ENV RUST_BACKTRACE=full +ENV RUST_LOG=full + +COPY tools /onefuzz/tools \ No newline at end of file diff --git a/src/docker/linux/build.sh b/src/docker/linux/build.sh new file mode 100755 index 0000000000..41e7837704 --- /dev/null +++ b/src/docker/linux/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cp -r ../../runtime-tools tools +(cd ../../../; ./src/ci/agent.sh; cp artifacts/agent/onefuzz-agent artifacts/agent/onefuzz-supervisor src/docker/linux/tools/linux) +(cd ../../../; ./src/ci/azcopy.sh; cp artifacts/azcopy/azcopy src/docker/linux/tools/linux) +docker build -t onefuzz:latest . \ No newline at end of file From a5352aa75ffb131321ce7ca32a19768c58641e4e Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 18:19:56 -0400 Subject: [PATCH 02/35] update supervisor to support config from env --- .gitignore | 2 +- src/agent/onefuzz-supervisor/.gitignore | 1 + src/agent/onefuzz-supervisor/src/config.rs | 41 ++++++++++++++++++++-- src/agent/onefuzz-supervisor/src/main.rs | 9 ++--- 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4681a90788..a93e85ab8e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,6 @@ .idea **/.direnv **/.envrc - +artifacts # vim *.swp diff --git a/src/agent/onefuzz-supervisor/.gitignore b/src/agent/onefuzz-supervisor/.gitignore index 96ef6c0b94..04eeebc316 100644 --- a/src/agent/onefuzz-supervisor/.gitignore +++ b/src/agent/onefuzz-supervisor/.gitignore @@ -1,2 +1,3 @@ /target Cargo.lock +data/licenses.json \ No newline at end of file diff --git a/src/agent/onefuzz-supervisor/src/config.rs b/src/agent/onefuzz-supervisor/src/config.rs index 72bc968c4a..c5afb447aa 100644 --- a/src/agent/onefuzz-supervisor/src/config.rs +++ b/src/agent/onefuzz-supervisor/src/config.rs @@ -68,8 +68,45 @@ impl StaticConfig { Ok(config) } - pub async fn load(config_path: impl AsRef) -> Result { - let data = tokio::fs::read(config_path).await?; + pub fn from_env() -> Result { + let client_id = Uuid::parse_str(&std::env::var("ONEFUZZ_CLIENT_ID")?)?; + let client_secret = std::env::var("ONEFUZZ_CLIENT_SECRET")?.into(); + let tenant = std::env::var("ONEFUZZ_TENANT")?; + let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; + let pool_name = std::env::var("ONEFUZZ_POOL")?; + + let instrumentation_key = if let Ok(key) = std::env::var("ONEFUZZ_INSTRUMENTATION_KEY") { + Some(Uuid::parse_str(&key)?) + } else { + None + }; + + let telemetry_key = if let Ok(key) = std::env::var("ONEFUZZ_TELEMETRY_KEY") { + Some(Uuid::parse_str(&key)?) + } else { + None + }; + + let credentials = ClientCredentials::new( + client_id, + client_secret, + onefuzz_url.clone().to_string(), + tenant, + ) + .into(); + + Ok(Self { + credentials, + pool_name, + onefuzz_url, + instrumentation_key, + telemetry_key, + }) + } + + pub fn from_file(config_path: impl AsRef) -> Result { + verbose!("loading config from: {}", config_path.as_ref().display()); + let data = std::fs::read(config_path)?; Self::new(&data) } diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 2334c81f52..7832926b73 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -45,7 +45,7 @@ enum Opt { #[derive(StructOpt, Debug)] struct RunOpt { #[structopt(short, long = "--config", parse(from_os_str))] - config_path: PathBuf, + config_path: Option, } fn main() -> Result<()> { @@ -108,9 +108,10 @@ fn run(opt: RunOpt) -> Result<()> { fn load_config(opt: RunOpt) -> Result { info!("loading supervisor agent config"); - let data = std::fs::read(&opt.config_path)?; - let config = StaticConfig::new(&data)?; - verbose!("loaded static config from: {}", opt.config_path.display()); + let config = match &opt.config_path { + Some(config_path) => StaticConfig::from_file(config_path)?, + None => StaticConfig::from_env()?, + }; init_telemetry(&config); From 5bc524350266300f34c033e11f28d5e64da70273 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 18:30:12 -0400 Subject: [PATCH 03/35] enable simple builds of the agent/supervisor when built locally --- src/docker/linux/build.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/docker/linux/build.sh b/src/docker/linux/build.sh index 41e7837704..f5a38d53a9 100755 --- a/src/docker/linux/build.sh +++ b/src/docker/linux/build.sh @@ -1,6 +1,17 @@ #!/bin/bash +rm -rf tools cp -r ../../runtime-tools tools -(cd ../../../; ./src/ci/agent.sh; cp artifacts/agent/onefuzz-agent artifacts/agent/onefuzz-supervisor src/docker/linux/tools/linux) -(cd ../../../; ./src/ci/azcopy.sh; cp artifacts/azcopy/azcopy src/docker/linux/tools/linux) -docker build -t onefuzz:latest . \ No newline at end of file + +if [ ! -d ../../../artifacts/azcopy ]; then + (cd ../../../; ./src/ci/agent.sh) +fi +(cd ../../../; cp artifacts/azcopy/azcopy src/docker/linux/tools/linux) + +if [ -d ../../../artifacts/agent ]; then + (cd ../../../; cp artifacts/agent/onefuzz-agent artifacts/agent/onefuzz-supervisor src/docker/linux/tools/linux) +else + (cd ../../agent; cargo build --release; cp target/release/onefuzz-{agent,supervisor} ../src/docker/linux/tools/linux) +fi + +docker build -t onefuzz:latest . From 7e71369e79a24bdafc2d85897df890c7de275991 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 18:55:39 -0400 Subject: [PATCH 04/35] for local testing, just build the agent, don't run the full CI script --- src/docker/linux/build.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/docker/linux/build.sh b/src/docker/linux/build.sh index f5a38d53a9..7d2bdeccd1 100755 --- a/src/docker/linux/build.sh +++ b/src/docker/linux/build.sh @@ -1,17 +1,20 @@ #!/bin/bash +BUILD_DIR=$(dirname $(realpath $0)) +cd ${BUILD_DIR} + rm -rf tools -cp -r ../../runtime-tools tools +cp -r ../../runtime-tools ${BUILD_DIR}/tools if [ ! -d ../../../artifacts/azcopy ]; then (cd ../../../; ./src/ci/agent.sh) fi -(cd ../../../; cp artifacts/azcopy/azcopy src/docker/linux/tools/linux) +(cd ../../../; cp artifacts/azcopy/azcopy ${BUILD_DIR}/tools/linux) if [ -d ../../../artifacts/agent ]; then - (cd ../../../; cp artifacts/agent/onefuzz-agent artifacts/agent/onefuzz-supervisor src/docker/linux/tools/linux) + (cd ../../../; cp artifacts/agent/onefuzz-agent artifacts/agent/onefuzz-supervisor ${BUILD_DIR}/tools/linux) else - (cd ../../agent; cargo build --release; cp target/release/onefuzz-{agent,supervisor} ../src/docker/linux/tools/linux) + (cd ../../agent; cargo build --release; cp target/release/onefuzz-{agent,supervisor} ${BUILD_DIR}/tools/linux) fi docker build -t onefuzz:latest . From 0d5a622e675923ae5348ba750b4ac7bbba23d2a1 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 21:21:34 -0400 Subject: [PATCH 05/35] continued dev --- src/agent/onefuzz-supervisor/src/auth.rs | 33 +++++-------------- src/agent/onefuzz-supervisor/src/main.rs | 10 ++++-- .../__app__/onefuzzlib/agent_authorization.py | 15 ++++++--- src/docker/linux/Dockerfile | 6 ++-- src/docker/linux/build.sh | 2 ++ 5 files changed, 31 insertions(+), 35 deletions(-) diff --git a/src/agent/onefuzz-supervisor/src/auth.rs b/src/agent/onefuzz-supervisor/src/auth.rs index 528ec1e930..b558d50475 100644 --- a/src/agent/onefuzz-supervisor/src/auth.rs +++ b/src/agent/onefuzz-supervisor/src/auth.rs @@ -104,13 +104,17 @@ impl ClientCredentials { let mut url = Url::parse("https://login.microsoftonline.com")?; url.path_segments_mut() .expect("Authority URL is cannot-be-a-base") - .push(&self.tenant) - .push("oauth2/v2.0/token"); + .extend(&[&self.tenant, "oauth2", "v2.0", "token"]); let response = reqwest::Client::new() .post(url) - .header("Content-Length", "0") - .form(&self.form_data()) + .form(&[ + ("client_id", self.client_id.to_hyphenated().to_string()), + ("client_secret", self.client_secret.expose_ref().to_string()), + ("grant_type", "client_credentials".into()), + ("tenant", self.tenant.clone()), + ("scope", format!("{}.default", self.resource)), + ]) .send() .await? .error_for_status()?; @@ -119,27 +123,6 @@ impl ClientCredentials { Ok(body.into()) } - - fn form_data(&self) -> FormData { - let scope = format!("{}/.default", self.resource); - - FormData { - client_id: self.client_id, - client_secret: self.client_secret.clone(), - grant_type: "client_credentials".into(), - scope, - tenant: self.tenant.clone(), - } - } -} - -#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)] -struct FormData { - client_id: Uuid, - client_secret: Secret, - grant_type: String, - scope: String, - tenant: String, } // See: https://docs.microsoft.com/en-us/azure/active-directory/develop diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 7832926b73..8a1a2fb47e 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -121,11 +121,15 @@ fn load_config(opt: RunOpt) -> Result { async fn run_agent(config: StaticConfig) -> Result<()> { telemetry::set_property(EventData::MachineId(get_machine_id().await?)); telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); - if let Ok(scaleset) = get_scaleset_name().await { + + let registration = if let Ok(scaleset) = get_scaleset_name().await { telemetry::set_property(EventData::ScalesetId(scaleset)); - } + config::Registration::create_managed(config.clone()).await? + } else { + config::Registration::create_unmanaged(config.clone()).await? + }; + - let registration = config::Registration::create_managed(config.clone()).await?; verbose!("created managed registration: {:?}", registration); let coordinator = coordinator::Coordinator::new(registration.clone()).await?; diff --git a/src/api-service/__app__/onefuzzlib/agent_authorization.py b/src/api-service/__app__/onefuzzlib/agent_authorization.py index 7eaf113293..d871ad8f93 100644 --- a/src/api-service/__app__/onefuzzlib/agent_authorization.py +++ b/src/api-service/__app__/onefuzzlib/agent_authorization.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import logging -from typing import Callable, Union +from typing import Callable, Union, Optional from uuid import UUID import azure.functions as func @@ -20,7 +20,7 @@ class TokenData(BaseModel): application_id: UUID - object_id: UUID + object_id: Optional[UUID] def try_get_token_auth_header(request: func.HttpRequest) -> Union[Error, TokenData]: @@ -49,12 +49,19 @@ def try_get_token_auth_header(request: func.HttpRequest) -> Union[Error, TokenDa # This token has already been verified by the azure authentication layer token = jwt.decode(parts[1], verify=False) - return TokenData(application_id=UUID(token["appid"]), object_id=UUID(token["oid"])) + + application_id = UUID("appid") if "appid" in token else None + object_id = UUID(token["oid"]) if "oid" in token else None + return TokenData(application_id=application_id, object_id=object_id) @cached(ttl=60) def is_authorized(token_data: TokenData) -> bool: - scalesets = Scaleset.get_by_object_id(token_data.object_id) + if token_data.object_id: + scalesets = Scaleset.get_by_object_id(token_data.object_id) + return len(scalesets) > 0 + + scalesets = Scaleset.get_by_object_id(token_data.application_id) return len(scalesets) > 0 diff --git a/src/docker/linux/Dockerfile b/src/docker/linux/Dockerfile index ef3e19f916..35ddc2246c 100644 --- a/src/docker/linux/Dockerfile +++ b/src/docker/linux/Dockerfile @@ -13,6 +13,6 @@ ENV ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer RUN ln -f -s $(which llvm-symbolizer-10) $ASAN_SYMBOLIZER_PATH ENV RUST_BACKTRACE=full -ENV RUST_LOG=full - -COPY tools /onefuzz/tools \ No newline at end of file +ENV RUST_LOG=debug +ENV PATH=/onefuzz/bin:/onefuzz/tools/linux:$PATH +COPY tools /onefuzz/tools diff --git a/src/docker/linux/build.sh b/src/docker/linux/build.sh index 7d2bdeccd1..7f2acf6d47 100755 --- a/src/docker/linux/build.sh +++ b/src/docker/linux/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -ex + BUILD_DIR=$(dirname $(realpath $0)) cd ${BUILD_DIR} From 7b3c14b7a49b23a42da4bf2625131a0c7a224853 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 21:24:00 -0400 Subject: [PATCH 06/35] cargo fmt --- src/agent/onefuzz-supervisor/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 8a1a2fb47e..f0ff330e1a 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -129,7 +129,6 @@ async fn run_agent(config: StaticConfig) -> Result<()> { config::Registration::create_unmanaged(config.clone()).await? }; - verbose!("created managed registration: {:?}", registration); let coordinator = coordinator::Coordinator::new(registration.clone()).await?; From d14c7d4bdbb6752e1192e40e799338dcc387fc9e Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 21:28:04 -0400 Subject: [PATCH 07/35] fix appid --- src/api-service/__app__/onefuzzlib/agent_authorization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api-service/__app__/onefuzzlib/agent_authorization.py b/src/api-service/__app__/onefuzzlib/agent_authorization.py index d871ad8f93..8b3f33b12f 100644 --- a/src/api-service/__app__/onefuzzlib/agent_authorization.py +++ b/src/api-service/__app__/onefuzzlib/agent_authorization.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import logging -from typing import Callable, Union, Optional +from typing import Callable, Optional, Union from uuid import UUID import azure.functions as func @@ -50,7 +50,7 @@ def try_get_token_auth_header(request: func.HttpRequest) -> Union[Error, TokenDa # This token has already been verified by the azure authentication layer token = jwt.decode(parts[1], verify=False) - application_id = UUID("appid") if "appid" in token else None + application_id = UUID(token["appid"]) object_id = UUID(token["oid"]) if "oid" in token else None return TokenData(application_id=application_id, object_id=object_id) From fa00dfbde85a4909469815ab7b56e35093246eff Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 21:59:42 -0400 Subject: [PATCH 08/35] continued dev --- src/agent/onefuzz-supervisor/src/config.rs | 6 +++--- src/api-service/__app__/onefuzzlib/agent_authorization.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/agent/onefuzz-supervisor/src/config.rs b/src/agent/onefuzz-supervisor/src/config.rs index c5afb447aa..d365334304 100644 --- a/src/agent/onefuzz-supervisor/src/config.rs +++ b/src/agent/onefuzz-supervisor/src/config.rs @@ -148,13 +148,13 @@ impl Registration { let mut url = config.register_url(); url.query_pairs_mut() .append_pair("machine_id", &machine_id.to_string()) - .append_pair("pool_name", &config.pool_name); + .append_pair("pool_name", &config.pool_name) + .append_pair("version", env!("ONEFUZZ_VERSION")); if managed { let scaleset = onefuzz::machine_id::get_scaleset_name().await?; url.query_pairs_mut() - .append_pair("scaleset_id", &scaleset) - .append_pair("version", env!("ONEFUZZ_VERSION")); + .append_pair("scaleset_id", &scaleset); } // The registration can fail because this call is made before the virtual machine scaleset is done provisioning // The authentication layer of the service will reject this request when that happens diff --git a/src/api-service/__app__/onefuzzlib/agent_authorization.py b/src/api-service/__app__/onefuzzlib/agent_authorization.py index 8b3f33b12f..b74956bf9e 100644 --- a/src/api-service/__app__/onefuzzlib/agent_authorization.py +++ b/src/api-service/__app__/onefuzzlib/agent_authorization.py @@ -14,7 +14,7 @@ from onefuzztypes.models import Error from pydantic import BaseModel -from .pools import Scaleset +from .pools import Scaleset, Pool from .request import not_ok @@ -61,8 +61,8 @@ def is_authorized(token_data: TokenData) -> bool: scalesets = Scaleset.get_by_object_id(token_data.object_id) return len(scalesets) > 0 - scalesets = Scaleset.get_by_object_id(token_data.application_id) - return len(scalesets) > 0 + pools = Pool.search(query={"client_id", token_data.application_id}) + return len(pools) > 0 def verify_token( From a26e570b08cfd7652ebcaab7963664507d34c768 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 22:02:38 -0400 Subject: [PATCH 09/35] cargo fmt --- src/agent/onefuzz-supervisor/src/config.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/agent/onefuzz-supervisor/src/config.rs b/src/agent/onefuzz-supervisor/src/config.rs index d365334304..1c4908802f 100644 --- a/src/agent/onefuzz-supervisor/src/config.rs +++ b/src/agent/onefuzz-supervisor/src/config.rs @@ -153,8 +153,7 @@ impl Registration { if managed { let scaleset = onefuzz::machine_id::get_scaleset_name().await?; - url.query_pairs_mut() - .append_pair("scaleset_id", &scaleset); + url.query_pairs_mut().append_pair("scaleset_id", &scaleset); } // The registration can fail because this call is made before the virtual machine scaleset is done provisioning // The authentication layer of the service will reject this request when that happens From 3d90b77571cbc4f3f3f7a95bf4e26d989069d8f6 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 22:15:10 -0400 Subject: [PATCH 10/35] isort --- src/api-service/__app__/onefuzzlib/agent_authorization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api-service/__app__/onefuzzlib/agent_authorization.py b/src/api-service/__app__/onefuzzlib/agent_authorization.py index b74956bf9e..6b49579bca 100644 --- a/src/api-service/__app__/onefuzzlib/agent_authorization.py +++ b/src/api-service/__app__/onefuzzlib/agent_authorization.py @@ -14,7 +14,7 @@ from onefuzztypes.models import Error from pydantic import BaseModel -from .pools import Scaleset, Pool +from .pools import Pool, Scaleset from .request import not_ok @@ -61,7 +61,7 @@ def is_authorized(token_data: TokenData) -> bool: scalesets = Scaleset.get_by_object_id(token_data.object_id) return len(scalesets) > 0 - pools = Pool.search(query={"client_id", token_data.application_id}) + pools = Pool.search(query={"client_id": [token_data.application_id]}) return len(pools) > 0 From df5feb23d400f630d71866dfe232d8efae18e3a4 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 22:15:20 -0400 Subject: [PATCH 11/35] ignore env-config --- src/docker/linux/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/docker/linux/.gitignore b/src/docker/linux/.gitignore index 71e9f312d4..5db856397e 100644 --- a/src/docker/linux/.gitignore +++ b/src/docker/linux/.gitignore @@ -1 +1,2 @@ -tools \ No newline at end of file +tools +env-config From 758f40770d83f9dddce0bdafc0eafb3bc8f6c76b Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 23:02:17 -0400 Subject: [PATCH 12/35] don't init app insights if it's optional --- src/agent/onefuzz-agent/src/main.rs | 14 ++------- src/agent/onefuzz-supervisor/src/main.rs | 11 ++----- src/agent/onefuzz/src/telemetry.rs | 37 ++++++++++++------------ 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 3d917de20b..129651a735 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -84,17 +84,7 @@ fn licenses() -> Result<()> { } fn init_telemetry(config: &Config) { - let inst_key = config - .common() - .instrumentation_key - .map(|k| k.to_string()) - .unwrap_or_else(|| "".to_string()); - - let tele_key = config - .common() - .telemetry_key - .map(|k| k.to_string()) - .unwrap_or_else(|| "".to_string()); - + let inst_key = config.common().instrumentation_key; + let tele_key = config.common().telemetry_key; telemetry::set_appinsights_clients(inst_key, tele_key); } diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index f0ff330e1a..55431d75ad 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -159,14 +159,7 @@ async fn run_agent(config: StaticConfig) -> Result<()> { } fn init_telemetry(config: &StaticConfig) { - let inst_key = config - .instrumentation_key - .map(|k| k.to_string()) - .unwrap_or_else(String::default); - let tele_key = config - .telemetry_key - .map(|k| k.to_string()) - .unwrap_or_else(String::default); - + let inst_key = config.instrumentation_key; + let tele_key = config.telemetry_key; telemetry::set_appinsights_clients(inst_key, tele_key); } diff --git a/src/agent/onefuzz/src/telemetry.rs b/src/agent/onefuzz/src/telemetry.rs index 795de1c016..816cf077e0 100644 --- a/src/agent/onefuzz/src/telemetry.rs +++ b/src/agent/onefuzz/src/telemetry.rs @@ -160,7 +160,7 @@ mod global { static STATE: AtomicUsize = AtomicUsize::new(UNSET); - pub fn set_clients(instance: TelemetryClient, shared: TelemetryClient) { + pub fn set_clients(instance: Option, shared: Option) { use Ordering::SeqCst; let last_state = STATE.compare_and_swap(UNSET, SETTING, SeqCst); @@ -176,9 +176,9 @@ mod global { assert_eq!(last_state, UNSET, "unexpected telemetry client state"); unsafe { - CLIENTS.instance = Some(RwLock::new(instance)); - CLIENTS.shared = Some(RwLock::new(shared)); - }; + CLIENTS.instance = instance.map(|s| RwLock::new(s)); + CLIENTS.shared = shared.map(|s| RwLock::new(s)); + } STATE.store(SET, SeqCst); } @@ -224,10 +224,9 @@ mod global { } } -pub fn set_appinsights_clients(ikey: impl Into, tkey: impl Into) { - let instance_client = TelemetryClient::new(ikey.into()); - let shared_client = TelemetryClient::new(tkey.into()); - +pub fn set_appinsights_clients(ikey: Option, tkey: Option) { + let instance_client = ikey.map(|k| TelemetryClient::new(k.to_string())); + let shared_client = tkey.map(|k| TelemetryClient::new(k.to_string())); global::set_clients(instance_client, shared_client); } @@ -272,7 +271,7 @@ pub fn try_client_mut( pub fn property(client_type: ClientType, key: impl AsRef) -> Option { let key = key.as_ref(); - let client = client(client_type).expect("telemetry client called internally when unset"); + let client = client(client_type).expect("telemetry client called internally when unset AA"); Some(client.context().properties().get(key)?.to_owned()) } @@ -281,20 +280,20 @@ pub fn set_property(entry: EventData) { let (key, value) = entry.as_values(); if entry.can_share() { - let mut client = - client_mut(ClientType::Shared).expect("telemetry client called internally when unset"); + if let Some(mut client) = client_mut(ClientType::Shared) { + client + .context_mut() + .properties_mut() + .insert(key.to_owned(), value.to_owned()); + } + } + + if let Some(mut client) = client_mut(ClientType::Instance) { client .context_mut() .properties_mut() - .insert(key.to_owned(), value.to_owned()); + .insert(key.to_owned(), value); } - - let mut client = - client_mut(ClientType::Instance).expect("telemetry client called internally when unset"); - client - .context_mut() - .properties_mut() - .insert(key.to_owned(), value); } pub fn track_event(event: Event, properties: Vec) { From 485f6de42a8ffc61b3faa84376ce47b41b4547f7 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 12 Oct 2020 23:30:03 -0400 Subject: [PATCH 13/35] add version command --- src/agent/onefuzz-supervisor/src/main.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 55431d75ad..59949c2bbd 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -40,6 +40,7 @@ enum Opt { Run(RunOpt), Debug(debug::DebugOpt), Licenses, + Version, } #[derive(StructOpt, Debug)] @@ -57,11 +58,22 @@ fn main() -> Result<()> { Opt::Run(opt) => run(opt)?, Opt::Debug(opt) => debug::debug(opt)?, Opt::Licenses => licenses()?, + Opt::Version => versions()?, }; Ok(()) } +fn versions() -> Result<()> { + println!( + "{} onefuzz:{} git:{}", + crate_version!(), + env!("ONEFUZZ_VERSION"), + env!("GIT_VERSION") + ); + Ok(()) +} + fn licenses() -> Result<()> { use std::io::{self, Write}; io::stdout().write_all(include_bytes!("../../data/licenses.json"))?; @@ -69,13 +81,6 @@ fn licenses() -> Result<()> { } fn run(opt: RunOpt) -> Result<()> { - info!( - "{} onefuzz:{} git:{}", - crate_version!(), - env!("ONEFUZZ_VERSION"), - env!("GIT_VERSION") - ); - if done::is_agent_done()? { verbose!( "agent is done, remove lock ({}) to continue", From 53d2b384b2e0afbcd726fcef56b6c8d3deb4a782 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 00:11:47 -0400 Subject: [PATCH 14/35] make /onefuzz/etc & start supervisor --- src/docker/linux/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/docker/linux/Dockerfile b/src/docker/linux/Dockerfile index 35ddc2246c..4f22d28b75 100644 --- a/src/docker/linux/Dockerfile +++ b/src/docker/linux/Dockerfile @@ -8,11 +8,14 @@ RUN apt-get update \ && apt-get install -y gdb gdbserver llvm-10 wget unzip libssl1.0.0 libunwind-dev \ && apt-get clean -RUN mkdir -p /onefuzz/bin +RUN mkdir -p /onefuzz/bin /onefuzz/etc ENV ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer RUN ln -f -s $(which llvm-symbolizer-10) $ASAN_SYMBOLIZER_PATH -ENV RUST_BACKTRACE=full +ENV RUST_BACKTRACE=full ENV RUST_LOG=debug ENV PATH=/onefuzz/bin:/onefuzz/tools/linux:$PATH COPY tools /onefuzz/tools + +WORKDIR /onefuzz +ENTRYPOINT ["onefuzz-supervisor", "run"] \ No newline at end of file From 91cf6d1bc81158016ef566feacfd1d4d1865e895 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 00:56:37 -0400 Subject: [PATCH 15/35] continued dev --- src/agent/onefuzz/src/telemetry.rs | 11 ++++++----- src/docker/linux/Dockerfile | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/agent/onefuzz/src/telemetry.rs b/src/agent/onefuzz/src/telemetry.rs index 816cf077e0..2660a850ba 100644 --- a/src/agent/onefuzz/src/telemetry.rs +++ b/src/agent/onefuzz/src/telemetry.rs @@ -269,11 +269,12 @@ pub fn try_client_mut( } pub fn property(client_type: ClientType, key: impl AsRef) -> Option { - let key = key.as_ref(); - - let client = client(client_type).expect("telemetry client called internally when unset AA"); - - Some(client.context().properties().get(key)?.to_owned()) + client(client_type).map(|c| { + c.context() + .properties() + .get(key.as_ref()) + .map(|s| s.to_owned()) + })? } pub fn set_property(entry: EventData) { diff --git a/src/docker/linux/Dockerfile b/src/docker/linux/Dockerfile index 4f22d28b75..a55b10463f 100644 --- a/src/docker/linux/Dockerfile +++ b/src/docker/linux/Dockerfile @@ -13,7 +13,7 @@ ENV ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer RUN ln -f -s $(which llvm-symbolizer-10) $ASAN_SYMBOLIZER_PATH ENV RUST_BACKTRACE=full -ENV RUST_LOG=debug +ENV RUST_LOG=info ENV PATH=/onefuzz/bin:/onefuzz/tools/linux:$PATH COPY tools /onefuzz/tools From 83eeb9c794e539132fd52d1a8fa153f1384f598a Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 01:05:01 -0400 Subject: [PATCH 16/35] put auth code in onefuzz (to be reused for agent-startup) --- src/agent/Cargo.lock | 1 + src/agent/onefuzz-supervisor/src/config.rs | 5 ++--- src/agent/onefuzz-supervisor/src/coordinator.rs | 6 ++++-- src/agent/onefuzz-supervisor/src/main.rs | 1 - src/agent/onefuzz-supervisor/src/work.rs | 6 ++++-- src/agent/onefuzz/Cargo.toml | 1 + src/agent/{onefuzz-supervisor => onefuzz}/src/auth.rs | 0 src/agent/onefuzz/src/lib.rs | 1 + 8 files changed, 13 insertions(+), 8 deletions(-) rename src/agent/{onefuzz-supervisor => onefuzz}/src/auth.rs (100%) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 507bb9107e..3ee13d5b73 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -1240,6 +1240,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", + "url", "urlparse", "uuid", "winreg", diff --git a/src/agent/onefuzz-supervisor/src/config.rs b/src/agent/onefuzz-supervisor/src/config.rs index 1c4908802f..c223103f63 100644 --- a/src/agent/onefuzz-supervisor/src/config.rs +++ b/src/agent/onefuzz-supervisor/src/config.rs @@ -1,18 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use reqwest::StatusCode; use std::{ path::Path, time::{Duration, Instant}, }; use anyhow::Result; +use onefuzz::auth::{ClientCredentials, Credentials, ManagedIdentityCredentials}; +use reqwest::StatusCode; use url::Url; use uuid::Uuid; -use crate::auth::{ClientCredentials, Credentials, ManagedIdentityCredentials}; - #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct StaticConfig { pub credentials: Credentials, diff --git a/src/agent/onefuzz-supervisor/src/coordinator.rs b/src/agent/onefuzz-supervisor/src/coordinator.rs index 8ad58c30b1..f718051ca2 100644 --- a/src/agent/onefuzz-supervisor/src/coordinator.rs +++ b/src/agent/onefuzz-supervisor/src/coordinator.rs @@ -3,12 +3,14 @@ use anyhow::Result; use downcast_rs::Downcast; -use onefuzz::process::Output; +use onefuzz::{ + process::Output, + auth::AccessToken, +}; use reqwest::{Client, Request, Response, StatusCode}; use serde::Serialize; use uuid::Uuid; -use crate::auth::AccessToken; use crate::config::Registration; use crate::work::{TaskId, WorkSet}; use crate::worker::WorkerEvent; diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 59949c2bbd..2b3173c399 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -22,7 +22,6 @@ use onefuzz::{ use structopt::StructOpt; pub mod agent; -pub mod auth; pub mod config; pub mod coordinator; pub mod debug; diff --git a/src/agent/onefuzz-supervisor/src/work.rs b/src/agent/onefuzz-supervisor/src/work.rs index da5aaf8801..a6d8885732 100644 --- a/src/agent/onefuzz-supervisor/src/work.rs +++ b/src/agent/onefuzz-supervisor/src/work.rs @@ -5,11 +5,13 @@ use std::path::PathBuf; use anyhow::Result; use downcast_rs::Downcast; -use onefuzz::blob::BlobContainerUrl; +use onefuzz::{ + blob::BlobContainerUrl, + auth::Secret, +}; use storage_queue::QueueClient; use uuid::Uuid; -use crate::auth::Secret; use crate::config::Registration; pub type JobId = Uuid; diff --git a/src/agent/onefuzz/Cargo.toml b/src/agent/onefuzz/Cargo.toml index 23e114f1e4..6f161e0f27 100644 --- a/src/agent/onefuzz/Cargo.toml +++ b/src/agent/onefuzz/Cargo.toml @@ -20,6 +20,7 @@ notify = "4.0" regex = "1.3" reqwest = { version = "0.10", features = ["json", "stream"] } sha2 = "0.9" +url = { version = "2.1.1", features = ["serde"] } serde = "1.0" serde_json = "1.0" serde_derive = "1.0" diff --git a/src/agent/onefuzz-supervisor/src/auth.rs b/src/agent/onefuzz/src/auth.rs similarity index 100% rename from src/agent/onefuzz-supervisor/src/auth.rs rename to src/agent/onefuzz/src/auth.rs diff --git a/src/agent/onefuzz/src/lib.rs b/src/agent/onefuzz/src/lib.rs index ee69d72ffd..57605a0228 100644 --- a/src/agent/onefuzz/src/lib.rs +++ b/src/agent/onefuzz/src/lib.rs @@ -14,6 +14,7 @@ extern crate serde; pub mod telemetry; pub mod asan; +pub mod auth; pub mod az_copy; pub mod blob; pub mod expand; From 2f6c738193c47ccdcc0e6faeb33ea95be03420a7 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 09:12:35 -0400 Subject: [PATCH 17/35] cargo fmt --- src/agent/onefuzz-supervisor/src/coordinator.rs | 5 +---- src/agent/onefuzz-supervisor/src/work.rs | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/agent/onefuzz-supervisor/src/coordinator.rs b/src/agent/onefuzz-supervisor/src/coordinator.rs index f718051ca2..a737370824 100644 --- a/src/agent/onefuzz-supervisor/src/coordinator.rs +++ b/src/agent/onefuzz-supervisor/src/coordinator.rs @@ -3,10 +3,7 @@ use anyhow::Result; use downcast_rs::Downcast; -use onefuzz::{ - process::Output, - auth::AccessToken, -}; +use onefuzz::{auth::AccessToken, process::Output}; use reqwest::{Client, Request, Response, StatusCode}; use serde::Serialize; use uuid::Uuid; diff --git a/src/agent/onefuzz-supervisor/src/work.rs b/src/agent/onefuzz-supervisor/src/work.rs index a6d8885732..b384af08c6 100644 --- a/src/agent/onefuzz-supervisor/src/work.rs +++ b/src/agent/onefuzz-supervisor/src/work.rs @@ -5,10 +5,7 @@ use std::path::PathBuf; use anyhow::Result; use downcast_rs::Downcast; -use onefuzz::{ - blob::BlobContainerUrl, - auth::Secret, -}; +use onefuzz::{auth::Secret, blob::BlobContainerUrl}; use storage_queue::QueueClient; use uuid::Uuid; From a1700a5461242e2d829f2bbad8a14bff6bb6593e Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 11:08:15 -0400 Subject: [PATCH 18/35] move to using a downloader to ensure the container is always in-sync --- src/agent/Cargo.lock | 23 ++++ src/agent/Cargo.toml | 1 + src/agent/onefuzz-downloader/.gitignore | 3 + src/agent/onefuzz-downloader/Cargo.toml | 26 ++++ src/agent/onefuzz-downloader/build.rs | 56 ++++++++ src/agent/onefuzz-downloader/src/config.rs | 104 +++++++++++++++ src/agent/onefuzz-downloader/src/main.rs | 124 ++++++++++++++++++ src/agent/onefuzz-downloader/src/setup.rs | 122 +++++++++++++++++ .../__app__/agent_download/__init__.py | 37 ++++++ .../__app__/agent_download/function.json | 20 +++ src/docker/linux/Dockerfile | 2 +- src/docker/linux/build.sh | 10 +- src/pytypes/onefuzztypes/requests.py | 6 + src/pytypes/onefuzztypes/responses.py | 4 + 14 files changed, 532 insertions(+), 6 deletions(-) create mode 100644 src/agent/onefuzz-downloader/.gitignore create mode 100644 src/agent/onefuzz-downloader/Cargo.toml create mode 100644 src/agent/onefuzz-downloader/build.rs create mode 100644 src/agent/onefuzz-downloader/src/config.rs create mode 100644 src/agent/onefuzz-downloader/src/main.rs create mode 100644 src/agent/onefuzz-downloader/src/setup.rs create mode 100644 src/api-service/__app__/agent_download/__init__.py create mode 100644 src/api-service/__app__/agent_download/function.json diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 3ee13d5b73..6bf3d48fd9 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -1272,6 +1272,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "onefuzz-downloader" +version = "0.1.0" +dependencies = [ + "anyhow", + "appinsights", + "async-trait", + "clap", + "downcast-rs", + "env_logger", + "futures", + "log", + "onefuzz", + "reqwest", + "serde", + "serde_json", + "storage-queue", + "structopt", + "tokio", + "url", + "uuid", +] + [[package]] name = "onefuzz-supervisor" version = "0.1.0" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index a486217126..cfaf322a80 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -5,6 +5,7 @@ members = [ "input-tester", "onefuzz", "onefuzz-agent", + "onefuzz-downloader", "onefuzz-supervisor", "storage-queue", "win-util", diff --git a/src/agent/onefuzz-downloader/.gitignore b/src/agent/onefuzz-downloader/.gitignore new file mode 100644 index 0000000000..04eeebc316 --- /dev/null +++ b/src/agent/onefuzz-downloader/.gitignore @@ -0,0 +1,3 @@ +/target +Cargo.lock +data/licenses.json \ No newline at end of file diff --git a/src/agent/onefuzz-downloader/Cargo.toml b/src/agent/onefuzz-downloader/Cargo.toml new file mode 100644 index 0000000000..ab28f40aa8 --- /dev/null +++ b/src/agent/onefuzz-downloader/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "onefuzz-downloader" +version = "0.1.0" +authors = ["fuzzing@microsoft.com"] +edition = "2018" +publish = false + + +[dependencies] +anyhow = "1.0.31" +appinsights = "0.1" +async-trait = "0.1.36" +downcast-rs = "1.2.0" +env_logger = "0.7" +futures = "0.3.5" +log = "0.4" +onefuzz = { path = "../onefuzz" } +reqwest = { version = "0.10", features = ["json", "stream"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +storage-queue = { path = "../storage-queue" } +structopt = "0.3" +tokio = { version = "0.2.13", features = ["full"] } +url = { version = "2.1.1", features = ["serde"] } +uuid = { version = "0.8.1", features = ["serde", "v4"] } +clap = "2.33" diff --git a/src/agent/onefuzz-downloader/build.rs b/src/agent/onefuzz-downloader/build.rs new file mode 100644 index 0000000000..bec63b58b2 --- /dev/null +++ b/src/agent/onefuzz-downloader/build.rs @@ -0,0 +1,56 @@ +use std::env; +use std::error::Error; +use std::fs::File; +use std::io::prelude::*; +use std::process::Command; + +fn run_cmd(args: &[&str]) -> Result> { + let cmd = Command::new(args[0]).args(&args[1..]).output()?; + if cmd.status.success() { + Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string()) + } else { + Err(From::from("failed")) + } +} + +fn read_file(filename: &str) -> Result> { + let mut file = File::open(filename)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + contents = contents.trim().to_string(); + + Ok(contents) +} + +fn print_version(include_sha: bool, include_local: bool) -> Result<(), Box> { + let mut version = read_file("../../../CURRENT_VERSION")?; + let sha = run_cmd(&["git", "rev-parse", "HEAD"])?; + + if include_sha { + version.push('-'); + version.push_str(&sha); + + // if we're a non-release build, check to see if git has + // unstaged changes + if include_local && run_cmd(&["git", "diff", "--quiet"]).is_err() { + version.push('.'); + version.push_str("localchanges"); + } + } + + println!("cargo:rustc-env=GIT_VERSION={}", sha); + println!("cargo:rustc-env=ONEFUZZ_VERSION={}", version); + + Ok(()) +} + +fn main() -> Result<(), Box> { + // If we're built off of a tag, we accept CURRENT_VERSION as is. Otherwise + // modify it to indicate local build + let (include_sha, include_local_changes) = if let Ok(val) = env::var("GITHUB_REF") { + (!val.starts_with("refs/tags/"), false) + } else { + (true, true) + }; + print_version(include_sha, include_local_changes) +} diff --git a/src/agent/onefuzz-downloader/src/config.rs b/src/agent/onefuzz-downloader/src/config.rs new file mode 100644 index 0000000000..09449c062a --- /dev/null +++ b/src/agent/onefuzz-downloader/src/config.rs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use anyhow::Result; +use onefuzz::auth::{ClientCredentials, Credentials, ManagedIdentityCredentials}; +use std::path::Path; +use url::Url; +use uuid::Uuid; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct StaticConfig { + pub credentials: Credentials, + pub pool_name: String, + pub onefuzz_url: Url, + pub instrumentation_key: Option, + pub telemetry_key: Option, +} + +// Temporary shim type to bridge the current service-provided config. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +struct RawStaticConfig { + pub credentials: Option, + pub pool_name: String, + pub onefuzz_url: Url, + pub instrumentation_key: Option, + pub telemetry_key: Option, +} + +impl StaticConfig { + pub fn new(data: &[u8]) -> Result { + let config: RawStaticConfig = serde_json::from_slice(data)?; + + let credentials = match config.credentials { + Some(client) => client.into(), + None => { + // Remove trailing `/`, which is treated as a distinct resource. + let resource = config + .onefuzz_url + .to_string() + .trim_end_matches('/') + .to_owned(); + let managed = ManagedIdentityCredentials::new(resource); + managed.into() + } + }; + let config = StaticConfig { + credentials, + pool_name: config.pool_name, + onefuzz_url: config.onefuzz_url, + instrumentation_key: config.instrumentation_key, + telemetry_key: config.telemetry_key, + }; + + Ok(config) + } + + pub fn from_env() -> Result { + let client_id = Uuid::parse_str(&std::env::var("ONEFUZZ_CLIENT_ID")?)?; + let client_secret = std::env::var("ONEFUZZ_CLIENT_SECRET")?.into(); + let tenant = std::env::var("ONEFUZZ_TENANT")?; + let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; + let pool_name = std::env::var("ONEFUZZ_POOL")?; + + let instrumentation_key = if let Ok(key) = std::env::var("ONEFUZZ_INSTRUMENTATION_KEY") { + Some(Uuid::parse_str(&key)?) + } else { + None + }; + + let telemetry_key = if let Ok(key) = std::env::var("ONEFUZZ_TELEMETRY_KEY") { + Some(Uuid::parse_str(&key)?) + } else { + None + }; + + let credentials = ClientCredentials::new( + client_id, + client_secret, + onefuzz_url.clone().to_string(), + tenant, + ) + .into(); + + Ok(Self { + credentials, + pool_name, + onefuzz_url, + instrumentation_key, + telemetry_key, + }) + } + + pub fn from_file(config_path: impl AsRef) -> Result { + verbose!("loading config from: {}", config_path.as_ref().display()); + let data = std::fs::read(config_path)?; + Self::new(&data) + } + + pub fn download_url(&self) -> Url { + let mut url = self.onefuzz_url.clone(); + url.set_path("/api/agents/download"); + url + } +} diff --git a/src/agent/onefuzz-downloader/src/main.rs b/src/agent/onefuzz-downloader/src/main.rs new file mode 100644 index 0000000000..bf00a55016 --- /dev/null +++ b/src/agent/onefuzz-downloader/src/main.rs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#[macro_use] +extern crate anyhow; +#[macro_use] +extern crate onefuzz; +#[macro_use] +extern crate serde; +#[macro_use] +extern crate clap; + +use std::path::PathBuf; + +use anyhow::Result; +use onefuzz::{ + machine_id::get_machine_id, + telemetry::{self, EventData}, +}; +use structopt::StructOpt; + +pub mod config; +pub mod setup; + +use config::StaticConfig; + +#[derive(StructOpt, Debug)] +enum Opt { + Run(RunOpt), + Licenses, + Version, +} + +#[derive(StructOpt, Debug)] +struct RunOpt { + #[structopt(short, long = "--config", parse(from_os_str))] + config_path: Option, + + #[structopt(short, long)] + start_supervisor: bool, +} + +fn main() -> Result<()> { + env_logger::init(); + + let opt = Opt::from_args(); + + match opt { + Opt::Run(opt) => run(opt)?, + Opt::Licenses => licenses()?, + Opt::Version => versions()?, + }; + + Ok(()) +} + +fn versions() -> Result<()> { + println!( + "{} onefuzz:{} git:{}", + crate_version!(), + env!("ONEFUZZ_VERSION"), + env!("GIT_VERSION") + ); + Ok(()) +} + +fn licenses() -> Result<()> { + use std::io::{self, Write}; + io::stdout().write_all(include_bytes!("../../data/licenses.json"))?; + Ok(()) +} + +fn run(opt: RunOpt) -> Result<()> { + // We can't send telemetry if this fails. + let config = load_config(&opt); + + if let Err(err) = &config { + error!("error loading supervisor agent config: {:?}", err); + } + + let config = config?; + init_telemetry(&config); + + let mut rt = tokio::runtime::Runtime::new()?; + let result = rt.block_on(run_downloader(config, &opt)); + + if let Err(err) = &result { + error!("error running downloader: {}", err); + } + + telemetry::try_flush_and_close(); + + result +} + +fn load_config(opt: &RunOpt) -> Result { + info!("loading downloader config"); + let config = match &opt.config_path { + Some(config_path) => StaticConfig::from_file(config_path)?, + None => StaticConfig::from_env()?, + }; + + Ok(config) +} + +async fn run_downloader(config: StaticConfig, opt: &RunOpt) -> Result<()> { + telemetry::set_property(EventData::MachineId(get_machine_id().await?)); + telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); + + let setup_inst = setup::Setup { config }; + setup_inst.run().await?; + + if opt.start_supervisor { + setup_inst.launch_supervisor(&opt.config_path).await?; + } + + Ok(()) +} + +fn init_telemetry(config: &StaticConfig) { + let inst_key = config.instrumentation_key; + let tele_key = config.telemetry_key; + telemetry::set_appinsights_clients(inst_key, tele_key); +} diff --git a/src/agent/onefuzz-downloader/src/setup.rs b/src/agent/onefuzz-downloader/src/setup.rs new file mode 100644 index 0000000000..9b20bcfed8 --- /dev/null +++ b/src/agent/onefuzz-downloader/src/setup.rs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::config::StaticConfig; +use anyhow::Result; +use onefuzz::az_copy; +use onefuzz::process::Output; +use std::path::PathBuf; +use std::process::Stdio; +use tokio::fs; +use tokio::process::Command; +use url::Url; + +pub struct Setup { + pub config: StaticConfig, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct DownloadConfig { + pub tools: Url, +} + +impl Setup { + pub async fn run(&self) -> Result<()> { + let download_config = self.get_download_config().await?; + let tools_dir = onefuzz::fs::onefuzz_root()?.join("tools"); + + fs::create_dir_all(&tools_dir).await?; + az_copy::sync(download_config.tools.to_string(), &tools_dir).await?; + let output: Output = self.setup_command(tools_dir).output().await?.into(); + + if output.exit_status.success { + verbose!( + "setup script succeeded. stdout:{:?}, stderr:{:?}", + output.stdout, + output.stderr + ); + } else { + bail!( + "setup script failed. stdout:{:?}, stderr:{:?}", + output.stdout, + output.stderr + ); + } + + Ok(()) + } + + pub async fn launch_supervisor(&self, path: &Option) -> Result<()> { + let mut cmd = Command::new("onefuzz-supervisor"); + cmd.arg("run"); + if let Some(path) = path { + cmd.arg("--config"); + cmd.arg(path); + } + + let output: Output = cmd.output().await?.into(); + if output.exit_status.success { + verbose!( + "supervisor succeeded. stdout:{:?}, stderr:{:?}", + output.stdout, + output.stderr + ); + } else { + bail!( + "supervisor failed. stdout:{:?}, stderr:{:?}", + output.stdout, + output.stderr + ); + } + Ok(()) + } + + #[cfg(target_os = "windows")] + fn setup_command(&self, mut path: PathBuf) -> Command { + path.push("win64"); + path.push("setup-download.ps1"); + + let mut cmd = Command::new("powershell.exe"); + cmd.env(SETUP_PATH_ENV, setup_script); + cmd.arg("-ExecutionPolicy"); + cmd.arg("Unrestricted"); + cmd.arg("-File"); + cmd.arg(&self.script_path); + cmd.stderr(Stdio::piped()); + cmd.stdout(Stdio::piped()); + + cmd + } + + #[cfg(target_os = "linux")] + fn setup_command(&self, mut path: PathBuf) -> Command { + path.push("linux"); + path.push("setup-download.sh"); + let mut cmd = Command::new("bash"); + cmd.arg(path); + cmd.stderr(Stdio::piped()); + cmd.stdout(Stdio::piped()); + + cmd + } + + async fn get_download_config(&self) -> Result { + let token = self.config.credentials.access_token().await?; + let machine_id = onefuzz::machine_id::get_machine_id().await?; + let mut url = self.config.download_url(); + url.query_pairs_mut() + .append_pair("machine_id", &machine_id.to_string()) + .append_pair("pool_name", &self.config.pool_name) + .append_pair("version", env!("ONEFUZZ_VERSION")); + + let response = reqwest::Client::new() + .get(url) + .bearer_auth(token.secret().expose_ref()) + .send() + .await? + .error_for_status()?; + + let to_download: DownloadConfig = response.json().await?; + Ok(to_download) + } +} diff --git a/src/api-service/__app__/agent_download/__init__.py b/src/api-service/__app__/agent_download/__init__.py new file mode 100644 index 0000000000..ce8e473ebb --- /dev/null +++ b/src/api-service/__app__/agent_download/__init__.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import azure.functions as func +from onefuzztypes.models import Error +from onefuzztypes.requests import DownloadConfigRequest +from onefuzztypes.responses import DownloadConfig + +from ..onefuzzlib.azure.containers import get_container_sas_url +from ..onefuzzlib.azure.creds import get_func_storage +from ..onefuzzlib.agent_authorization import verify_token +from ..onefuzzlib.pools import Node +from ..onefuzzlib.request import not_ok, ok, parse_request +from ..onefuzzlib.tasks.main import Task + + +def get(req: func.HttpRequest) -> func.HttpResponse: + request = parse_request(DownloadConfigRequest, req) + if isinstance(request, Error): + return not_ok(request, context="DownloadConfigRequest") + + tools_sas = get_container_sas_url( + "tools", read=True, list=True, account_id=get_func_storage() + ) + + return ok(DownloadConfig(tools=tools_sas)) + + +def main(req: func.HttpRequest) -> func.HttpResponse: + if req.method == "GET": + m = get + else: + raise Exception("invalid method") + + return verify_token(req, m) \ No newline at end of file diff --git a/src/api-service/__app__/agent_download/function.json b/src/api-service/__app__/agent_download/function.json new file mode 100644 index 0000000000..4b87ffb43d --- /dev/null +++ b/src/api-service/__app__/agent_download/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get" + ], + "route": "agents/download" + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/src/docker/linux/Dockerfile b/src/docker/linux/Dockerfile index a55b10463f..4ed271f165 100644 --- a/src/docker/linux/Dockerfile +++ b/src/docker/linux/Dockerfile @@ -18,4 +18,4 @@ ENV PATH=/onefuzz/bin:/onefuzz/tools/linux:$PATH COPY tools /onefuzz/tools WORKDIR /onefuzz -ENTRYPOINT ["onefuzz-supervisor", "run"] \ No newline at end of file +ENTRYPOINT ["onefuzz-downloader", "run", "--start-supervisor"] \ No newline at end of file diff --git a/src/docker/linux/build.sh b/src/docker/linux/build.sh index 7f2acf6d47..215db1c74f 100755 --- a/src/docker/linux/build.sh +++ b/src/docker/linux/build.sh @@ -6,17 +6,17 @@ BUILD_DIR=$(dirname $(realpath $0)) cd ${BUILD_DIR} rm -rf tools -cp -r ../../runtime-tools ${BUILD_DIR}/tools +mkdir -p tools/linux if [ ! -d ../../../artifacts/azcopy ]; then - (cd ../../../; ./src/ci/agent.sh) + (cd ../../../; ./src/ci/agent.sh) fi (cd ../../../; cp artifacts/azcopy/azcopy ${BUILD_DIR}/tools/linux) if [ -d ../../../artifacts/agent ]; then - (cd ../../../; cp artifacts/agent/onefuzz-agent artifacts/agent/onefuzz-supervisor ${BUILD_DIR}/tools/linux) + (cd ../../../; cp artifacts/agent/onefuzz-downloader ${BUILD_DIR}/tools/linux) else - (cd ../../agent; cargo build --release; cp target/release/onefuzz-{agent,supervisor} ${BUILD_DIR}/tools/linux) + (cd ../../agent; cargo build --release; cp target/release/onefuzz-downloader ${BUILD_DIR}/tools/linux) fi -docker build -t onefuzz:latest . +docker build -t onefuzz:latest . \ No newline at end of file diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index 023bb2dd3d..24fff3b874 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -203,3 +203,9 @@ class ProxyReset(BaseRequest): class CanScheduleRequest(BaseRequest): machine_id: UUID task_id: UUID + + +class DownloadConfigRequest(BaseRequest): + machine_id: UUID + pool_name: str + version: str \ No newline at end of file diff --git a/src/pytypes/onefuzztypes/responses.py b/src/pytypes/onefuzztypes/responses.py index 19acd55723..44d4f6a3fa 100644 --- a/src/pytypes/onefuzztypes/responses.py +++ b/src/pytypes/onefuzztypes/responses.py @@ -63,3 +63,7 @@ class PendingNodeCommand(BaseResponse): class CanSchedule(BaseResponse): allowed: bool work_stopped: bool + + +class DownloadConfig(BaseResponse): + tools: str \ No newline at end of file From fc83bbcbc035341c02ad4ea0b912df9349af5583 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 11:18:53 -0400 Subject: [PATCH 19/35] fix lint --- src/api-service/__app__/agent_download/__init__.py | 6 ++---- src/pytypes/onefuzztypes/requests.py | 2 +- src/pytypes/onefuzztypes/responses.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/api-service/__app__/agent_download/__init__.py b/src/api-service/__app__/agent_download/__init__.py index ce8e473ebb..348614e872 100644 --- a/src/api-service/__app__/agent_download/__init__.py +++ b/src/api-service/__app__/agent_download/__init__.py @@ -8,12 +8,10 @@ from onefuzztypes.requests import DownloadConfigRequest from onefuzztypes.responses import DownloadConfig +from ..onefuzzlib.agent_authorization import verify_token from ..onefuzzlib.azure.containers import get_container_sas_url from ..onefuzzlib.azure.creds import get_func_storage -from ..onefuzzlib.agent_authorization import verify_token -from ..onefuzzlib.pools import Node from ..onefuzzlib.request import not_ok, ok, parse_request -from ..onefuzzlib.tasks.main import Task def get(req: func.HttpRequest) -> func.HttpResponse: @@ -34,4 +32,4 @@ def main(req: func.HttpRequest) -> func.HttpResponse: else: raise Exception("invalid method") - return verify_token(req, m) \ No newline at end of file + return verify_token(req, m) diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index 24fff3b874..be74c55d6a 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -208,4 +208,4 @@ class CanScheduleRequest(BaseRequest): class DownloadConfigRequest(BaseRequest): machine_id: UUID pool_name: str - version: str \ No newline at end of file + version: str diff --git a/src/pytypes/onefuzztypes/responses.py b/src/pytypes/onefuzztypes/responses.py index 44d4f6a3fa..384d4902ff 100644 --- a/src/pytypes/onefuzztypes/responses.py +++ b/src/pytypes/onefuzztypes/responses.py @@ -66,4 +66,4 @@ class CanSchedule(BaseResponse): class DownloadConfig(BaseResponse): - tools: str \ No newline at end of file + tools: str From 80db79c10664e112d270eef633f73a1765cd4549 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 11:19:01 -0400 Subject: [PATCH 20/35] add downloader to ci --- .github/workflows/ci.yml | 2 ++ src/ci/agent.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6d5588ed9..e9e7b33e35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,8 +244,10 @@ jobs: cp artifacts/azcopy/azcopy.exe artifacts/azcopy/ThirdPartyNotice.txt src/deployment/tools/win64 cp artifacts/agent/onefuzz-supervisor.exe src/deployment/tools/win64/ cp artifacts/agent/onefuzz-agent.exe src/deployment/tools/win64/ + cp artifacts/agent/onefuzz-downloader.exe src/deployment/tools/win64/ cp artifacts/agent/onefuzz-supervisor src/deployment/tools/linux/ cp artifacts/agent/onefuzz-agent src/deployment/tools/linux/ + cp artifacts/agent/onefuzz-downloader src/deployment/tools/linux/ cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/ cp artifacts/service/api-service.zip src/deployment cp -r artifacts/third-party src/deployment diff --git a/src/ci/agent.sh b/src/ci/agent.sh index 642294a01d..0f14197d63 100755 --- a/src/ci/agent.sh +++ b/src/ci/agent.sh @@ -26,3 +26,4 @@ cargo test --release --manifest-path ./onefuzz/Cargo.toml cp target/release/onefuzz-agent* ../../artifacts/agent cp target/release/onefuzz-supervisor* ../../artifacts/agent +cp target/release/onefuzz-downloader* ../../artifacts/agent From d2d7a6a5578aba422283ea5dcbcc4f3e31822ef7 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 11:38:17 -0400 Subject: [PATCH 21/35] fix windows path --- src/agent/onefuzz-downloader/src/setup.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/agent/onefuzz-downloader/src/setup.rs b/src/agent/onefuzz-downloader/src/setup.rs index 9b20bcfed8..737283e04b 100644 --- a/src/agent/onefuzz-downloader/src/setup.rs +++ b/src/agent/onefuzz-downloader/src/setup.rs @@ -77,11 +77,10 @@ impl Setup { path.push("setup-download.ps1"); let mut cmd = Command::new("powershell.exe"); - cmd.env(SETUP_PATH_ENV, setup_script); cmd.arg("-ExecutionPolicy"); cmd.arg("Unrestricted"); cmd.arg("-File"); - cmd.arg(&self.script_path); + cmd.arg(path); cmd.stderr(Stdio::piped()); cmd.stdout(Stdio::piped()); From 591705dbfa70363f20d456d0d302636b2c431a46 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 13:49:06 -0400 Subject: [PATCH 22/35] use parse_uri instead of parse_request --- src/api-service/__app__/agent_download/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api-service/__app__/agent_download/__init__.py b/src/api-service/__app__/agent_download/__init__.py index 348614e872..a6b5bc3c21 100644 --- a/src/api-service/__app__/agent_download/__init__.py +++ b/src/api-service/__app__/agent_download/__init__.py @@ -11,11 +11,11 @@ from ..onefuzzlib.agent_authorization import verify_token from ..onefuzzlib.azure.containers import get_container_sas_url from ..onefuzzlib.azure.creds import get_func_storage -from ..onefuzzlib.request import not_ok, ok, parse_request +from ..onefuzzlib.request import not_ok, ok, parse_uri def get(req: func.HttpRequest) -> func.HttpResponse: - request = parse_request(DownloadConfigRequest, req) + request = parse_uri(DownloadConfigRequest, req) if isinstance(request, Error): return not_ok(request, context="DownloadConfigRequest") From d9e138c5120d9150607c5d43e0f6d0c6fb7c4740 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 14:02:47 -0400 Subject: [PATCH 23/35] custom install path --- src/agent/onefuzz-downloader/src/main.rs | 5 ++++- src/agent/onefuzz-downloader/src/setup.rs | 25 +++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/agent/onefuzz-downloader/src/main.rs b/src/agent/onefuzz-downloader/src/main.rs index bf00a55016..1b91b5c645 100644 --- a/src/agent/onefuzz-downloader/src/main.rs +++ b/src/agent/onefuzz-downloader/src/main.rs @@ -36,6 +36,9 @@ struct RunOpt { #[structopt(short, long = "--config", parse(from_os_str))] config_path: Option, + #[structopt(short, long = "--onefuzz_path", parse(from_os_str))] + onefuzz_path: Option, + #[structopt(short, long)] start_supervisor: bool, } @@ -108,7 +111,7 @@ async fn run_downloader(config: StaticConfig, opt: &RunOpt) -> Result<()> { telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); let setup_inst = setup::Setup { config }; - setup_inst.run().await?; + setup_inst.run(&opt.onefuzz_path).await?; if opt.start_supervisor { setup_inst.launch_supervisor(&opt.config_path).await?; diff --git a/src/agent/onefuzz-downloader/src/setup.rs b/src/agent/onefuzz-downloader/src/setup.rs index 737283e04b..f22686bdcd 100644 --- a/src/agent/onefuzz-downloader/src/setup.rs +++ b/src/agent/onefuzz-downloader/src/setup.rs @@ -5,8 +5,7 @@ use crate::config::StaticConfig; use anyhow::Result; use onefuzz::az_copy; use onefuzz::process::Output; -use std::path::PathBuf; -use std::process::Stdio; +use std::{env, path::PathBuf, process::Stdio}; use tokio::fs; use tokio::process::Command; use url::Url; @@ -21,9 +20,16 @@ pub struct DownloadConfig { } impl Setup { - pub async fn run(&self) -> Result<()> { + pub async fn run(&self, onefuzz_path: &Option) -> Result<()> { let download_config = self.get_download_config().await?; - let tools_dir = onefuzz::fs::onefuzz_root()?.join("tools"); + + let tools_dir = if let Some(onefuzz_path) = onefuzz_path { + onefuzz_path.join("tools") + } else { + onefuzz::fs::onefuzz_root()?.join("tools") + }; + + self.add_tools_path(&tools_dir)?; fs::create_dir_all(&tools_dir).await?; az_copy::sync(download_config.tools.to_string(), &tools_dir).await?; @@ -46,6 +52,17 @@ impl Setup { Ok(()) } + fn add_tools_path(&self, path: &PathBuf) -> Result<()> { + let path_env = env::var("PATH")?; + let os_path = match env::consts::OS { + "linux" => format!("{}:{}", path_env, path.join("linux").to_string_lossy()), + "windows" => format!("{};{}", path_env, path.join("win64").to_string_lossy()), + _ => unimplemented!("unsupported OS"), + }; + env::set_var("PATH", os_path); + Ok(()) + } + pub async fn launch_supervisor(&self, path: &Option) -> Result<()> { let mut cmd = Command::new("onefuzz-supervisor"); cmd.arg("run"); From ce3c9f8303c8740d0b9d13d484c090cabe5cc537 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 14:35:38 -0400 Subject: [PATCH 24/35] add basic setup-download.sh script --- src/runtime-tools/linux/setup-download.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 src/runtime-tools/linux/setup-download.sh diff --git a/src/runtime-tools/linux/setup-download.sh b/src/runtime-tools/linux/setup-download.sh new file mode 100755 index 0000000000..f98d7832d7 --- /dev/null +++ b/src/runtime-tools/linux/setup-download.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -x +mkdir -p /onefuzz/{bin,logs,tools,etc} +echo fuzz > /onefuzz/etc/mode +chmod -R a+rx /onefuzz/{bin,tools/linux} +/onefuzz/tools/linux/run.sh \ No newline at end of file From 19636741c477073e83161c1332efde56c4524fec Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 15:40:58 -0400 Subject: [PATCH 25/35] don't fail if we can't alter the environment --- src/runtime-tools/linux/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime-tools/linux/run.sh b/src/runtime-tools/linux/run.sh index 486136c3b0..6041c4ad5f 100755 --- a/src/runtime-tools/linux/run.sh +++ b/src/runtime-tools/linux/run.sh @@ -14,9 +14,9 @@ logger "onefuzz: starting up onefuzz" echo 0 | sudo tee /proc/sys/kernel/randomize_va_space # use core files, not external crash handler -echo core | sudo tee /proc/sys/kernel/core_pattern -echo 0 | sudo tee /proc/sys/kernel/randomize_va_space -echo 1 | sudo tee /proc/sys/fs/suid_dumpable +echo core | sudo tee /proc/sys/kernel/core_pattern || echo unable to set core pattern +echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || echo unable to disable ASLR +echo 1 | sudo tee /proc/sys/fs/suid_dumpable || echo unable to set suid_dumpable if type apt > /dev/null 2> /dev/null; then sudo apt update From a6716aa75a2aae5eab64dcafaa1866abe70267e3 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 15:43:13 -0400 Subject: [PATCH 26/35] move more install tasks into setup.sh --- src/runtime-tools/linux/run.sh | 19 +------------------ src/runtime-tools/linux/setup.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/runtime-tools/linux/run.sh b/src/runtime-tools/linux/run.sh index 6041c4ad5f..faa1509ecb 100755 --- a/src/runtime-tools/linux/run.sh +++ b/src/runtime-tools/linux/run.sh @@ -7,6 +7,7 @@ set -ex export PATH=$PATH:/onefuzz/bin:/onefuzz/tools/linux:/onefuzz/tools/linux/afl:/onefuzz/tools/linux/radamsa export ONEFUZZ_TOOLS=/onefuzz/tools export ONEFUZZ_ROOT=/onefuzz +export ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer logger "onefuzz: starting up onefuzz" @@ -18,11 +19,6 @@ echo core | sudo tee /proc/sys/kernel/core_pattern || echo unable to set core pa echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || echo unable to disable ASLR echo 1 | sudo tee /proc/sys/fs/suid_dumpable || echo unable to set suid_dumpable -if type apt > /dev/null 2> /dev/null; then - sudo apt update - sudo apt install -y gdb -fi - cd /onefuzz MODE=$(cat /onefuzz/etc/mode) case ${MODE} in @@ -36,24 +32,11 @@ case ${MODE} in "fuzz") logger "onefuzz: starting fuzzing" echo fuzzing - if type apt > /dev/null 2> /dev/null; then - export ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer - if ! [ -f ${ASAN_SYMBOLIZER_PATH} ]; then - sudo apt install -y llvm-10 - - # If specifying symbolizer, exe name must be a "known symbolizer". - # Using `llvm-symbolizer` works for clang 8 .. 10. - sudo ln -f -s $(which llvm-symbolizer-10) $ASAN_SYMBOLIZER_PATH - fi - fi export RUST_BACKTRACE=full onefuzz-supervisor run --config /onefuzz/config.json ;; "repro") logger "onefuzz: starting repro" - if type apt > /dev/null 2> /dev/null; then - sudo apt install -y gdb gdbserver - fi export ASAN_OPTIONS=abort_on_error=1 repro.sh ;; diff --git a/src/runtime-tools/linux/setup.sh b/src/runtime-tools/linux/setup.sh index d30fc4409c..65398ae84f 100755 --- a/src/runtime-tools/linux/setup.sh +++ b/src/runtime-tools/linux/setup.sh @@ -84,6 +84,19 @@ fi chmod -R a+rx /onefuzz/tools/linux +if type apt > /dev/null 2> /dev/null; then + sudo apt update + sudo apt install -y gdb gdbserver + + export ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer + if ! [ -f ${ASAN_SYMBOLIZER_PATH} ]; then + sudo apt install -y llvm-10 + + # If specifying symbolizer, exe name must be a "known symbolizer". + # Using `llvm-symbolizer` works for clang 8 .. 10. + sudo ln -f -s $(which llvm-symbolizer-10) $ASAN_SYMBOLIZER_PATH + fi +fi if [ -d /etc/systemd/system ]; then logger "onefuzz: setting up systemd" From 875ca9ed27f74088289de150df6557dc2c08e31c Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 17:06:34 -0400 Subject: [PATCH 27/35] use configurable onefuzz_path --- src/agent/onefuzz-downloader/src/setup.rs | 22 +++++++++++++++------- src/runtime-tools/linux/run.sh | 9 +++------ src/runtime-tools/linux/setup-download.sh | 14 ++++++++++---- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/agent/onefuzz-downloader/src/setup.rs b/src/agent/onefuzz-downloader/src/setup.rs index f22686bdcd..54a579b712 100644 --- a/src/agent/onefuzz-downloader/src/setup.rs +++ b/src/agent/onefuzz-downloader/src/setup.rs @@ -23,17 +23,23 @@ impl Setup { pub async fn run(&self, onefuzz_path: &Option) -> Result<()> { let download_config = self.get_download_config().await?; - let tools_dir = if let Some(onefuzz_path) = onefuzz_path { - onefuzz_path.join("tools") - } else { - onefuzz::fs::onefuzz_root()?.join("tools") + let onefuzz_path = match onefuzz_path { + Some(x) => x.to_owned(), + None => onefuzz::fs::onefuzz_root()?, }; + let tools_dir = onefuzz_path.join("tools"); self.add_tools_path(&tools_dir)?; fs::create_dir_all(&tools_dir).await?; + env::set_current_dir(&onefuzz_path)?; + az_copy::sync(download_config.tools.to_string(), &tools_dir).await?; - let output: Output = self.setup_command(tools_dir).output().await?.into(); + let output: Output = self + .setup_command(&onefuzz_path, tools_dir) + .output() + .await? + .into(); if output.exit_status.success { verbose!( @@ -89,7 +95,7 @@ impl Setup { } #[cfg(target_os = "windows")] - fn setup_command(&self, mut path: PathBuf) -> Command { + fn setup_command(&self, onefuzz_path: &PathBuf, mut path: PathBuf) -> Command { path.push("win64"); path.push("setup-download.ps1"); @@ -98,6 +104,7 @@ impl Setup { cmd.arg("Unrestricted"); cmd.arg("-File"); cmd.arg(path); + cmd.env("ONEFUZZ_ROOT", onefuzz_path); cmd.stderr(Stdio::piped()); cmd.stdout(Stdio::piped()); @@ -105,11 +112,12 @@ impl Setup { } #[cfg(target_os = "linux")] - fn setup_command(&self, mut path: PathBuf) -> Command { + fn setup_command(&self, onefuzz_path: &PathBuf, mut path: PathBuf) -> Command { path.push("linux"); path.push("setup-download.sh"); let mut cmd = Command::new("bash"); cmd.arg(path); + cmd.env("ONEFUZZ_ROOT", onefuzz_path); cmd.stderr(Stdio::piped()); cmd.stdout(Stdio::piped()); diff --git a/src/runtime-tools/linux/run.sh b/src/runtime-tools/linux/run.sh index faa1509ecb..4204788073 100755 --- a/src/runtime-tools/linux/run.sh +++ b/src/runtime-tools/linux/run.sh @@ -11,13 +11,10 @@ export ASAN_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer logger "onefuzz: starting up onefuzz" -# disable ASLR -echo 0 | sudo tee /proc/sys/kernel/randomize_va_space - # use core files, not external crash handler -echo core | sudo tee /proc/sys/kernel/core_pattern || echo unable to set core pattern -echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || echo unable to disable ASLR -echo 1 | sudo tee /proc/sys/fs/suid_dumpable || echo unable to set suid_dumpable +echo core | sudo tee /proc/sys/kernel/core_pattern +echo 0 | sudo tee /proc/sys/kernel/randomize_va_space +echo 1 | sudo tee /proc/sys/fs/suid_dumpable cd /onefuzz MODE=$(cat /onefuzz/etc/mode) diff --git a/src/runtime-tools/linux/setup-download.sh b/src/runtime-tools/linux/setup-download.sh index f98d7832d7..18c05c875b 100755 --- a/src/runtime-tools/linux/setup-download.sh +++ b/src/runtime-tools/linux/setup-download.sh @@ -3,7 +3,13 @@ # Licensed under the MIT License. set -x -mkdir -p /onefuzz/{bin,logs,tools,etc} -echo fuzz > /onefuzz/etc/mode -chmod -R a+rx /onefuzz/{bin,tools/linux} -/onefuzz/tools/linux/run.sh \ No newline at end of file + +export ONEFUZZ_TOOLS=${ONEFUZZ_ROOT}/tools +export ASAN_SYMBOLIZER_PATH=${ONEFUZZ_ROOT}/bin/llvm-symbolizer + +mkdir -p ${ONEFUZZ_ROOT}/{bin,logs,tools,etc} +chmod -R a+rx ${ONEFUZZ_ROOT}/{bin,tools/linux} + +echo core | sudo tee /proc/sys/kernel/core_pattern || echo unable to set core pattern +echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || echo unable to disable ASLR +echo 1 | sudo tee /proc/sys/fs/suid_dumpable || echo unable to set suid_dumpable \ No newline at end of file From 506460fa9d0c974d7ce49f9819c519598d339d36 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 17:46:57 -0400 Subject: [PATCH 28/35] don't consume stdout/stderr from supervisor --- src/agent/onefuzz-downloader/src/setup.rs | 16 +++------------- src/docker/linux/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/agent/onefuzz-downloader/src/setup.rs b/src/agent/onefuzz-downloader/src/setup.rs index 54a579b712..148d51cdf2 100644 --- a/src/agent/onefuzz-downloader/src/setup.rs +++ b/src/agent/onefuzz-downloader/src/setup.rs @@ -77,19 +77,9 @@ impl Setup { cmd.arg(path); } - let output: Output = cmd.output().await?.into(); - if output.exit_status.success { - verbose!( - "supervisor succeeded. stdout:{:?}, stderr:{:?}", - output.stdout, - output.stderr - ); - } else { - bail!( - "supervisor failed. stdout:{:?}, stderr:{:?}", - output.stdout, - output.stderr - ); + let output = cmd.status().await?; + if !output.success() { + bail!("supervisor failed: {:?}", output.code()); } Ok(()) } diff --git a/src/docker/linux/Dockerfile b/src/docker/linux/Dockerfile index 4ed271f165..dbfc8047a4 100644 --- a/src/docker/linux/Dockerfile +++ b/src/docker/linux/Dockerfile @@ -5,7 +5,7 @@ LABEL ABOUT="OneFuzz fuzzing container" ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get upgrade -y \ - && apt-get install -y gdb gdbserver llvm-10 wget unzip libssl1.0.0 libunwind-dev \ + && apt-get install -y gdb gdbserver llvm-10 wget unzip libssl1.0.0 libunwind-dev sudo \ && apt-get clean RUN mkdir -p /onefuzz/bin /onefuzz/etc From ae402a543b006db1405a6e604ab981d1069ff856 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 17:51:44 -0400 Subject: [PATCH 29/35] fix log message --- src/agent/onefuzz-supervisor/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 2b3173c399..d6741187f3 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -153,7 +153,7 @@ async fn run_agent(config: StaticConfig) -> Result<()> { Box::new(worker::WorkerRunner), ); - info!("running supervisor agent"); + info!("running agent"); agent.run().await?; From 33cee2cc611ffb80b324e8bea67589f08da74332 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 17:54:56 -0400 Subject: [PATCH 30/35] install github3 prereq --- src/api-service/__app__/requirements.txt | Bin 1772 -> 1804 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/api-service/__app__/requirements.txt b/src/api-service/__app__/requirements.txt index 7fecd83f1dae7bd51130b767d674b11c4ad0c98d..dc9b4cee33964c78f476c43c8a957c0820410a6f 100644 GIT binary patch delta 51 zcmaFE+rzgZgH1V|A(NqmA%mfmA&J45!4?P&8T5cGJq7~?UIs1(Wd?=G2iYVix3FaZ E03xgkM*si- delta 19 acmeC-d&9dSgN<34L1FSkHp$6L*fIb=4hAp) From 5be47a58a91e3d17fdb3bc063da97e5fa6f273d5 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 18:07:12 -0400 Subject: [PATCH 31/35] fix requirements --- src/api-service/__app__/requirements.txt | Bin 1804 -> 1804 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/api-service/__app__/requirements.txt b/src/api-service/__app__/requirements.txt index dc9b4cee33964c78f476c43c8a957c0820410a6f..f27e717330284de78735f353482ac1b4f34197ec 100644 GIT binary patch delta 21 ccmeC->*3o_#>T40P{2?*xs)w?@*_4i07gm%mjD0& delta 26 hcmeC->*3o_#x}WzEsIfk@2L}KE From c581e587626fbba516f55f07b04685f393512485 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 18:19:37 -0400 Subject: [PATCH 32/35] split name to prevent double inclusion --- src/api-service/__app__/requirements.txt | Bin 1804 -> 1806 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/api-service/__app__/requirements.txt b/src/api-service/__app__/requirements.txt index f27e717330284de78735f353482ac1b4f34197ec..d18fc5fecdebbb5266ba6f95b5db1fc2a662c989 100644 GIT binary patch delta 13 UcmeC->*L$7fQ?aM@_e>j03R6yQvd(} delta 11 ScmeC<>*3q5fNk<3wj2N%Wdt+; From 8884a6cf018ae408ed7a213766113e446e0d2337 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 19:25:50 -0400 Subject: [PATCH 33/35] dos2unix --- src/api-service/__app__/requirements.txt | 70 ++++++++++++------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/api-service/__app__/requirements.txt b/src/api-service/__app__/requirements.txt index deb7995d4d..551cfd8b3d 100644 --- a/src/api-service/__app__/requirements.txt +++ b/src/api-service/__app__/requirements.txt @@ -1,36 +1,36 @@ -azure-cli-core~=2.10.0 -azure-cli-nspkg==3.0.4 -azure-common~=1.1.25 -azure-core~=1.7.0 -azure-cosmosdb-nspkg==2.0.2 -azure-cosmosdb-table==1.0.6 -azure-devops==6.0.0b2 -azure-functions~=1.3.0 -azure-graphrbac~=0.61.1 -azure-identity~=1.3.1 -azure-keyvault-keys~=4.1.0 -azure-keyvault-secrets~=4.1.0 -azure-keyvault~=4.1.0 -azure-mgmt-compute~=13.0.0 -azure-mgmt-core~=1.0.0 -azure-mgmt-cosmosdb~=0.16.0 -azure-mgmt-loganalytics~=0.7.0 -azure-mgmt-network~=11.0.0 -azure-mgmt-resource~=10.1.0 -azure-mgmt-storage~=11.1.0 -azure-mgmt-subscription~=0.6.0 -azure-nspkg==3.0.2 -azure-servicebus==0.50.3 -azure-storage-blob==2.0.1 -azure-storage-common==2.1.0 -azure-storage-queue==12.1.2 -jinja2~=2.11.2 -msrestazure~=0.6.3 -opencensus-ext-azure~=1.0.2 -pydantic~=1.6.1 -PyJWT~=1.7.1 -requests~=2.24.0 -memoization~=0.3.1 -github3.py==1.3.0 -# onefuzz types version is set during build +azure-cli-core~=2.10.0 +azure-cli-nspkg==3.0.4 +azure-common~=1.1.25 +azure-core~=1.7.0 +azure-cosmosdb-nspkg==2.0.2 +azure-cosmosdb-table==1.0.6 +azure-devops==6.0.0b2 +azure-functions~=1.3.0 +azure-graphrbac~=0.61.1 +azure-identity~=1.3.1 +azure-keyvault-keys~=4.1.0 +azure-keyvault-secrets~=4.1.0 +azure-keyvault~=4.1.0 +azure-mgmt-compute~=13.0.0 +azure-mgmt-core~=1.0.0 +azure-mgmt-cosmosdb~=0.16.0 +azure-mgmt-loganalytics~=0.7.0 +azure-mgmt-network~=11.0.0 +azure-mgmt-resource~=10.1.0 +azure-mgmt-storage~=11.1.0 +azure-mgmt-subscription~=0.6.0 +azure-nspkg==3.0.2 +azure-servicebus==0.50.3 +azure-storage-blob==2.0.1 +azure-storage-common==2.1.0 +azure-storage-queue==12.1.2 +jinja2~=2.11.2 +msrestazure~=0.6.3 +opencensus-ext-azure~=1.0.2 +pydantic~=1.6.1 +PyJWT~=1.7.1 +requests~=2.24.0 +memoization~=0.3.1 +github3.py==1.3.0 +# onefuzz types version is set during build onefuzztypes==0.0.0 \ No newline at end of file From a29fcd9382b395ef50fe3cb5912afaecb6be7ce8 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 13 Oct 2020 19:26:14 -0400 Subject: [PATCH 34/35] use ~ 1.3.0 --- src/api-service/__app__/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api-service/__app__/requirements.txt b/src/api-service/__app__/requirements.txt index 551cfd8b3d..ffc792689c 100644 --- a/src/api-service/__app__/requirements.txt +++ b/src/api-service/__app__/requirements.txt @@ -31,6 +31,6 @@ pydantic~=1.6.1 PyJWT~=1.7.1 requests~=2.24.0 memoization~=0.3.1 -github3.py==1.3.0 +github3.py~=1.3.0 # onefuzz types version is set during build -onefuzztypes==0.0.0 \ No newline at end of file +onefuzztypes==0.0.0 From d085d14c031889ee20e43eae0e8c97660f497e2f Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 19 Oct 2020 09:55:33 -0400 Subject: [PATCH 35/35] address merge issues --- src/agent/onefuzz-supervisor/src/config.rs | 7 +++++++ src/agent/onefuzz-supervisor/src/coordinator.rs | 2 +- src/agent/onefuzz-supervisor/src/main.rs | 4 ++-- src/agent/onefuzz/src/auth.rs | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/agent/onefuzz-supervisor/src/config.rs b/src/agent/onefuzz-supervisor/src/config.rs index d5eb99c4e4..fd1173462d 100644 --- a/src/agent/onefuzz-supervisor/src/config.rs +++ b/src/agent/onefuzz-supervisor/src/config.rs @@ -80,6 +80,12 @@ impl StaticConfig { let onefuzz_url = Url::parse(&std::env::var("ONEFUZZ_URL")?)?; let pool_name = std::env::var("ONEFUZZ_POOL")?; + let heartbeat_queue = if let Ok(key) = std::env::var("ONEFUZZ_HEARTBEAT") { + Some(Url::parse(&key)?) + } else { + None + }; + let instrumentation_key = if let Ok(key) = std::env::var("ONEFUZZ_INSTRUMENTATION_KEY") { Some(Uuid::parse_str(&key)?) } else { @@ -106,6 +112,7 @@ impl StaticConfig { onefuzz_url, instrumentation_key, telemetry_key, + heartbeat_queue, }) } diff --git a/src/agent/onefuzz-supervisor/src/coordinator.rs b/src/agent/onefuzz-supervisor/src/coordinator.rs index 455a4d9d6d..790055cac4 100644 --- a/src/agent/onefuzz-supervisor/src/coordinator.rs +++ b/src/agent/onefuzz-supervisor/src/coordinator.rs @@ -3,7 +3,7 @@ use anyhow::Result; use downcast_rs::Downcast; -use onefuzz::{http::ResponseExt, process::Output}; +use onefuzz::{auth::AccessToken, http::ResponseExt, process::Output}; use reqwest::{Client, Request, Response, StatusCode}; use serde::Serialize; use uuid::Uuid; diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 151c01247e..09ec9992c9 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -128,8 +128,8 @@ async fn run_agent(config: StaticConfig) -> Result<()> { telemetry::set_property(EventData::MachineId(get_machine_id().await?)); telemetry::set_property(EventData::Version(env!("ONEFUZZ_VERSION").to_string())); let scaleset = get_scaleset_name().await; - if let Ok(scaleset) = scaleset { - telemetry::set_property(EventData::ScalesetId(scaleset)); + if let Ok(scaleset) = &scaleset { + telemetry::set_property(EventData::ScalesetId(scaleset.clone())); } let registration = match config::Registration::load_existing(config.clone()).await { diff --git a/src/agent/onefuzz/src/auth.rs b/src/agent/onefuzz/src/auth.rs index 8a0ab27fef..fab450e5a9 100644 --- a/src/agent/onefuzz/src/auth.rs +++ b/src/agent/onefuzz/src/auth.rs @@ -4,10 +4,11 @@ use std::fmt; use anyhow::Result; -use onefuzz::http::ResponseExt; use url::Url; use uuid::Uuid; +use crate::http::ResponseExt; + #[derive(Clone, Deserialize, Eq, PartialEq, Serialize)] pub struct Secret(T);