Merge branch 'main' into tevoinea/EnableScribanOnlyFF

microsoft · Feb 14, 2023 · 79dff70 · 79dff70
2 parents 9bc5d0e + 0bc22a8
commit 79dff70
Show file tree

Hide file tree

Showing 27 changed files with 588 additions and 138 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 6.4.0
+
+### Added 
+* Deployment/CLI: OneFuzz Config refactored - `tenant_id`, `tenant_domain`, `multi_tenant_domain`, and `cli_client_id` are now required values in the config.json used during deployment and no longer required when running the config command. [#2771](https://github.com/microsoft/onefuzz/pull/2771), [#2811](https://github.com/microsoft/onefuzz/pull/2811)
+* Agent: Fully escape allowlist rules [#2784](https://github.com/microsoft/onefuzz/pull/2784)
+* Agent: Apply allowlist to all blocks within a function [#2785](https://github.com/microsoft/onefuzz/pull/2785)
+* CLI: Added a cli subcommand `onefuzz debug notification template` to validate scriban notification templates [#2800](https://github.com/microsoft/onefuzz/pull/2800)
+* Service: Added Notification failure webhook to communicate Notification failures [#2628](https://github.com/microsoft/onefuzz/pull/2628)
+### Changed
+* Service: Include `AssignedTo` when failing to create a work item due to an authentication exception [#2770](https://github.com/microsoft/onefuzz/pull/2770)
+
+### Fixed
+* Agent: Fixes & improvements to `Expand` behavior [#2789](https://github.com/microsoft/onefuzz/pull/2789)
+* Agent: Triming whitespace in output from monitored process before printing [#2782](https://github.com/microsoft/onefuzz/pull/2782)
+* CLI: Fixed default value of analyzer_exe [#2797](https://github.com/microsoft/onefuzz/pull/2797)
+* CLI: Fixed missing `readonly_inputs` parameter in dotnet & dotnet_dll templates [#2740](https://github.com/microsoft/onefuzz/pull/2740)
+* Service: Fixed query to get the existing proxy [#2791](https://github.com/microsoft/onefuzz/pull/2791)
+* Service: Truncate webhooks message length for azure queue size compatibility [#2788](https://github.com/microsoft/onefuzz/pull/2788)
+
 ## 6.3.0
 ### Added 
 * Service: Add Optional Analysis Task to Libfuzzer Template [#2748](https://github.com/microsoft/onefuzz/pull/2748)

diff --git a/CURRENT_VERSION b/CURRENT_VERSION
@@ -1 +1 @@
-6.3.0
+6.4.0
diff --git a/docs/declarative-templates.md → docs/deprecated/declarative-templates.md b/docs/declarative-templates.md → docs/deprecated/declarative-templates.md
@@ -1,4 +1,4 @@
-# Declarative Job Templates
+# Declarative Job Templates -- Deprecated
 
 Provide the ability to maintain job templates, akin to `onefuzz template
 libfuzzer basic` at the service level. The templates include a job
@@ -94,7 +94,7 @@ TaskConfig(
 
 ## Hardcoded vs Runtime-specified Container Names
 
-To support differentiating _always use "afl-linux" for tools_ vs _ask 
+To support differentiating _always use "afl-linux" for tools_ vs _ask
 what container to use for setup_, if the container name is blank in the
 template, it will be provided as part of the `JobTemplateConfig` and in the
 resulting `JobTemplateRequest`.

diff --git a/docs/managing-templates.md → docs/deprecated/managing-templates.md b/docs/managing-templates.md → docs/deprecated/managing-templates.md
@@ -1,12 +1,12 @@
-# Managing Declarative Job Templates
+# Managing Declarative Job Templates -- Deprecated
 
 [Declarative Job Templates](declarative-templates.md), currently a preview
 feature, allow a user to define a reusable fuzzing pipeline as a template.
 Once saved, any user of the OneFuzz instance can create fuzzing jobs based on
 the templates.
 
 This is a walk-through guide for updating an existing job template, though
-the process is similar for creating templates from scratch. 
+the process is similar for creating templates from scratch.
 
 This process demonstrates adding [Microsoft Teams
 notifications](notifications/teams.md) for new unique crash reports to an existing
@@ -27,9 +27,9 @@ notifications](notifications/teams.md) for new unique crash reports to an existi
     onefuzz job_templates manage get libfuzzer_linux > libfuzzer_linux.json
     ```
 3. With your preferred text editor, add the following to the `notifications` list:
-    ```json 
+    ```json
     {
-        "container_type": "unique_reports", 
+        "container_type": "unique_reports",
         "notification": {
             "config": {
                 "url": "https://contoso.com/webhook-url-here"
@@ -120,7 +120,7 @@ Now let's make use of our new template.
         "job_id": "d3259dfe-fdad-45a0-bf90-a381b8dc1ee8",
         "state": "init"
     }
-    $ 
+    $
     ```
 3. Verify a notification was set up for the unique reports container
     ```
@@ -160,7 +160,7 @@ This example will demonstrate setting the following:
     onefuzz job_templates manage get libfuzzer_linux > libfuzzer_linux_ado_areapath.json
     ```
 3. With your preferred text editor, add the following to the `notifications` list:
-    ```json 
+    ```json
     {
         "container_type": "unique_reports",
         "notification": {
@@ -234,7 +234,7 @@ This example will demonstrate setting the following:
     onefuzz job_templates refresh
     ```
 
-Using `--help`, we can see the new optional and required arguments.  
+Using `--help`, we can see the new optional and required arguments.
 ```
 $ onefuzz job_templates submit libfuzzer_linux_ado_areapath --help
 usage: onefuzz job_templates submit libfuzzer_linux_ado_areapath [-h] [-v] [--format {json,raw}] [--query QUERY]

diff --git a/docs/unmnaged-nodes.md b/docs/unmnaged-nodes.md
@@ -87,4 +87,34 @@ onefuzz nodes get <machine_guid>
 ```
 
 This should return one entry. Verify that the `pool_name` matched the pool name created earlier.
-From here you will be able to schedule jobs on that pool and they will be running.
+From here you will be able to schedule jobs on that pool and they will be running.
+
+
+## Troubleshooting
+
+### increase the verbosity of the logs
+It can help when investigating issues to increase the log verbosity. you will need to set the [RUST_LOG](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) environment variable when starting docker
+
+```
+docker run --rm --env RUST_LOG=<log_level> <image_name> --machine_id <machine_id>
+```
+log_level can be any of
+- error
+- warn
+- info
+- debug
+- trace
+
+
+### use the container interactively
+you can use the container interactively by with the following command
+
+windows
+```
+docker run --it --rm --entrypoint powershell <image_name>
+```
+
+linux
+```
+docker run --it --rm --entrypoint bash <image_name>
+```
diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs
@@ -320,10 +320,6 @@ public async Async.Task<ResultVoid<TaskConfigError>> CheckConfig(TaskConfig conf
             return ResultVoid<TaskConfigError>.Error(new TaskConfigError($"invalid pool: {config.Pool.PoolName}"));
         }
 
-        if ((config.Task.RebootAfterSetup ?? false) && !pool.OkV.Managed) {
-            return ResultVoid<TaskConfigError>.Error(new TaskConfigError("reboot_after_setup is not supported for unmanaged pools"));
-        }
-
         var checkTarget = await CheckTargetExe(config, definition);
         if (!checkTarget.IsOk) {
             return checkTarget;

diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock
diff --git a/src/agent/cobertura/Cargo.toml b/src/agent/cobertura/Cargo.toml
@@ -6,4 +6,4 @@ license = "MIT"
 
 [dependencies]
 anyhow = "1.0"
-quick-xml = "0.26.0"
+quick-xml = "0.27.1"
diff --git a/src/agent/coverage-legacy/Cargo.toml b/src/agent/coverage-legacy/Cargo.toml
@@ -27,7 +27,7 @@ serde = { version = "1.0", features = ["derive"] }
 symbolic = { version = "8.8", features = ["debuginfo", "demangle", "symcache"] }
 uuid = { version = "0.8", features = ["guid"] }
 win-util = { path = "../win-util" }
-quick-xml = "0.26"
+quick-xml = "0.27"
 
 [target.'cfg(target_os = "windows")'.dependencies]
 pdb = "0.7"

diff --git a/src/agent/coverage/src/binary.rs b/src/agent/coverage/src/binary.rs
@@ -144,7 +144,6 @@ pub fn find_coverage_sites(
                     offsets.insert(block.offset);
                 }
             }
-            println!();
         }
     }
 

diff --git a/src/agent/input-tester/Cargo.toml b/src/agent/input-tester/Cargo.toml
@@ -12,7 +12,7 @@ debugger = { path = "../debugger" }
 fnv = "1.0"
 hex = "0.4"
 log = "0.4"
-num_cpus = "1.13"
+num_cpus = "1.15"
 rayon = "1.6"
 sha2 = "0.10.2"
 win-util = { path = "../win-util" }

diff --git a/src/agent/onefuzz-agent/src/reboot.rs b/src/agent/onefuzz-agent/src/reboot.rs
@@ -90,17 +90,22 @@ impl Reboot {
 
     #[cfg(target_family = "unix")]
     pub fn invoke(&self) -> Result<()> {
-        info!("invoking local reboot command");
-
-        Command::new("reboot").arg("-f").status()?;
-
-        self.wait_for_reboot()
+        match std::path::Path::new("/.dockerenv").try_exists() {
+            Ok(true) => {
+                info!("running inside docker, exiting instead of rebooting");
+                std::process::exit(0);
+            }
+            _ => {
+                info!("invoking local reboot command");
+                Command::new("reboot").arg("-f").status()?;
+                self.wait_for_reboot()
+            }
+        }
     }
 
     #[cfg(target_family = "windows")]
     pub fn invoke(&self) -> Result<()> {
         info!("invoking local reboot command");
-
         Command::new("powershell.exe")
             .arg("-Command")
             .arg("Restart-Computer")

diff --git a/src/agent/onefuzz-file-format/Cargo.toml b/src/agent/onefuzz-file-format/Cargo.toml
@@ -8,7 +8,7 @@ license = "MIT"
 anyhow = "1.0"
 coverage = { path = "../coverage" }
 debuggable-module = { path = "../debuggable-module" }
-quick-xml = "0.26.0"
+quick-xml = "0.27.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["preserve_order"] }
 

diff --git a/src/agent/onefuzz-task/Cargo.toml b/src/agent/onefuzz-task/Cargo.toml
@@ -24,7 +24,7 @@ futures = "0.3"
 hex = "0.4"
 lazy_static = "1.4"
 log = "0.4"
-num_cpus = "1.13"
+num_cpus = "1.15"
 regex = "1.6.0"
 reqwest = { version = "0.11", features = [
     "json",
@@ -55,13 +55,13 @@ chrono = { version = "0.4", default-features = false, features = [
     "std",
 ] }
 
-azure_core = { version = "0.6", default-features = false, features = [
+azure_core = { version = "0.8", default-features = false, features = [
     "enable_reqwest",
 ] }
-azure_storage = { version = "0.7", default-features = false, features = [
+azure_storage = { version = "0.9", default-features = false, features = [
     "enable_reqwest",
 ] }
-azure_storage_blobs = { version = "0.7", default-features = false, features = [
+azure_storage_blobs = { version = "0.9", default-features = false, features = [
     "enable_reqwest",
 ] }
 

diff --git a/src/agent/srcview/Cargo.toml b/src/agent/srcview/Cargo.toml
@@ -15,7 +15,7 @@ nom = "7"
 pdb = "0.8"
 regex = "1"
 serde = { version = "1", features = ["derive"] }
-quick-xml = "0.26"
+quick-xml = "0.27"
 anyhow = "1.0"
 structopt = "0.3"
 env_logger = "0.9"
diff --git a/src/agent/storage-queue/Cargo.toml b/src/agent/storage-queue/Cargo.toml
@@ -13,7 +13,7 @@ base64 = "0.13"
 bytes = { version = "1.2", features = ["serde"] }
 derivative = "2.2"
 flume = "0.10"
-num_cpus = "1.13"
+num_cpus = "1.15"
 regex = "1.6.0"
 reqwest = { version = "0.11", features = ["json", "stream", "native-tls-vendored"], default-features=false }
 reqwest-retry = { path = "../reqwest-retry" }

diff --git a/src/agent/win-util/Cargo.toml b/src/agent/win-util/Cargo.toml
@@ -9,7 +9,7 @@ license = "MIT"
 anyhow = "1.0"
 atexit = { path = "../atexit" }
 log = "0.4"
-os_pipe = "1.0"
+os_pipe = "1.1"
 
 [target.'cfg(windows)'.dependencies]
 winreg = "0.10"
-Original file line number
+Diff line change
@@ Expand Up / @@ -144,7 +144,6 @@ pub fn find_coverage_sites( @@
                         offsets.insert(block.offset);
                     }
                 }
-                println!();
             }
         }
@@ Expand Down @@