diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7384568ec..cd1872c36f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -527,7 +527,7 @@ jobs: uses: actions/cache@v3 with: path: src/integration-tests/artifacts - key: integration-tests|linux|${{ hashFiles('src/integration-tests/**/*') }} + key: integration-tests|linux|${{ env.ACTIONS_CACHE_KEY_DATE }}|${{ hashFiles('src/integration-tests/**/*') }} - name: Build integration tests if: steps.cache-integration-tests.outputs.cache-hit != 'true' run: | @@ -590,7 +590,7 @@ jobs: name: artifact-integration-tests-linux path: src/integration-tests/artifacts build-integration-tests-windows: - runs-on: windows-2019 + runs-on: windows-2022 steps: - uses: actions/checkout@v3 - name: Cache integration tests @@ -598,7 +598,9 @@ jobs: uses: actions/cache@v3 with: path: src/integration-tests/artifacts - key: integration-tests|windows|${{ hashFiles('src/integration-tests/**/*') }} + key: integration-tests|windows|${{ env.ACTIONS_CACHE_KEY_DATE }}|${{ hashFiles('src/integration-tests/**/*') }} + - name: Setup C/C++ environment + uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # pinned to v1.12.1 - name: Build integration tests if: steps.cache-integration-tests.outputs.cache-hit != 'true' run: | @@ -609,33 +611,6 @@ jobs: choco install make $env:Path += ";C:\Program Files\LLVM\bin;C:\ProgramData\chocolatey\bin" - # WORKAROUND: effectively downgrade the default Windows 10 SDK version. - # - # This ensures we link against a version of the SDK which won't trigger a - # startup bug in the LLVM-shipped ASAN runtime. - - # Assume a default MSVC 2019 install path. - $MsvcDir = 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC' - - # Assume that `$MsvcDir` only contains version-named subdirectories. - $MsvcVersion = ((Get-ChildItem $MsvcDir).name | Sort-Object -Descending)[0] - $MsvcLib = "${MsvcDir}/${MsvcVersion}/lib/x64" - - # Known "good" (non-bug-surfacing) version. - $WindowsSdkVersion = '10.0.18362.0' - - # Assume default install path. - $WindowsSdkDir = 'C:/Program Files (x86)/Windows Kits/10' - $WindowsSdkLib = "${WindowsSdkDir}/Lib/${WindowsSdkVersion}" - $WindowsSdkInclude = "${WindowsSdkDir}/Include/${WindowsSdkVersion}" - - # Used by `clang.exe`. - $env:CPATH = $WindowsSdkInclude - $env:LIBRARY_PATH = "${MsvcLib};${WindowsSdkLib}/ucrt/x64;${WindowsSdkLib}/um/x64" - - # Used by `link.exe`. - $env:LIB = $env:LIBRARY_PATH - cd src/integration-tests mkdir artifacts/windows-libfuzzer diff --git a/docs/webhook_events.md b/docs/webhook_events.md index ff97fa0c67..633703fb85 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -143,6 +143,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", @@ -1956,6 +1957,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", @@ -2867,6 +2869,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", @@ -3358,6 +3361,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", @@ -3867,6 +3871,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", @@ -4324,6 +4329,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", @@ -4808,6 +4814,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", @@ -5422,6 +5429,7 @@ If webhook is set to have Event Grid message format then the payload will look a "coverage", "crashes", "inputs", + "crashdumps", "no_repro", "readonly_inputs", "reports", diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index bd636caa19..97c840ba7f 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -106,6 +106,7 @@ public enum ContainerType { Analysis, Coverage, Crashes, + Crashdumps, Inputs, NoRepro, ReadonlyInputs, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 284a8b8368..e430c1448c 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -1103,6 +1103,7 @@ Dictionary Tags public IContainerDef? Analysis { get; set; } public IContainerDef? Coverage { get; set; } public IContainerDef? Crashes { get; set; } + public IContainerDef? Crashdumps { get; set; } public IContainerDef? Inputs { get; set; } public IContainerDef? NoRepro { get; set; } public IContainerDef? ReadonlyInputs { get; set; } diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs index 194ba51165..71af317348 100644 --- a/src/ApiService/ApiService/onefuzzlib/Config.cs +++ b/src/ApiService/ApiService/onefuzzlib/Config.cs @@ -118,6 +118,9 @@ await _containers.GetContainerSasUrl(container.Name, StorageType.Corpus, Convert case ContainerType.Crashes: config.Crashes = def; break; + case ContainerType.Crashdumps: + config.Crashdumps = def; + break; case ContainerType.Inputs: config.Inputs = def; break; diff --git a/src/ApiService/ApiService/onefuzzlib/Defs.cs b/src/ApiService/ApiService/onefuzzlib/Defs.cs index a1811a4f03..9a2337f1b7 100644 --- a/src/ApiService/ApiService/onefuzzlib/Defs.cs +++ b/src/ApiService/ApiService/onefuzzlib/Defs.cs @@ -190,6 +190,12 @@ public static class Defs { Value: 1, Permissions: ContainerPermission.Write ), + new ContainerDefinition( + Type:ContainerType.Crashdumps, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Write + ), new ContainerDefinition( Type: ContainerType.Inputs, Compare: Compare.Equal, @@ -279,6 +285,12 @@ public static class Defs { Value: 1, Permissions: ContainerPermission.Write ), + new ContainerDefinition( + Type:ContainerType.Crashdumps, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Write + ), new ContainerDefinition( Type: ContainerType.Inputs, Compare: Compare.Equal, diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 5b5ea2f11b..55cb816a09 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -10,11 +10,11 @@ checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" [[package]] name = "addr2line" -version = "0.17.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ - "gimli 0.26.2", + "gimli", ] [[package]] @@ -181,9 +181,9 @@ version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.13", + "proc-macro2 1.0.66", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -287,15 +287,15 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.63" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321629d8ba6513061f26707241fa9bc89524ff1cd7a915a97ef0c62c666ce1b6" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide 0.4.4", + "miniz_oxide", "object", "rustc-demangle", ] @@ -339,8 +339,8 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "regex", "rustc-hash", "shlex", @@ -358,18 +358,6 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" -[[package]] -name = "bitvec" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "block-buffer" version = "0.10.1" @@ -433,9 +421,12 @@ checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" [[package]] name = "cc" -version = "1.0.70" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +dependencies = [ + "libc", +] [[package]] name = "cexpr" @@ -513,9 +504,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" dependencies = [ "heck", - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.13", + "proc-macro2 1.0.66", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -613,9 +604,9 @@ dependencies = [ [[package]] name = "cpp_demangle" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b446fd40bcc17eddd6a4a78f24315eb90afdb3334999ddfd4909985c47722442" +checksum = "ee34052ee3d93d6d8f3e6f81d85c47921f6653a19a7b70e939e3e602d893a674" dependencies = [ "cfg-if 1.0.0", ] @@ -773,10 +764,10 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "scratch", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -791,9 +782,9 @@ version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.13", + "proc-macro2 1.0.66", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -822,8 +813,8 @@ dependencies = [ "anyhow", "clap", "elsa", - "gimli 0.27.2", - "goblin 0.6.0", + "gimli", + "goblin", "iced-x86", "log", "pdb", @@ -838,7 +829,7 @@ version = "0.1.0" dependencies = [ "anyhow", "fnv", - "goblin 0.5.1", + "goblin", "iced-x86", "log", "memmap2 0.7.1", @@ -854,6 +845,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ + "serde", "uuid", ] @@ -863,8 +855,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -887,9 +879,9 @@ dependencies = [ [[package]] name = "dmsort" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4699f5cb7678f099b747ffdc1e6ce6cdc42579e29d28315aa715b9fd10324fc" +checksum = "f0bc8fbe9441c17c9f46f75dfe27fa1ddb6c68a461ccaed0481419219d4f10d3" [[package]] name = "doc-comment" @@ -937,9 +929,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "elementtree" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e91f812124e4fc7f82605feda4da357307185a4619baee415ae0b7f6d5e031" +checksum = "3efd4742acf458718a6456e0adf0b4d734d6b783e452bbf1ac36bf31f4085cb3" dependencies = [ "string_cache", ] @@ -981,6 +973,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.2.8" @@ -1054,12 +1052,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", - "miniz_oxide 0.5.4", + "miniz_oxide", ] [[package]] @@ -1122,8 +1120,8 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63f713f8b2aa9e24fec85b0e290c56caee12e3b6ae0aeeda238a75b28251afd6" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -1164,12 +1162,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" version = "0.3.21" @@ -1239,8 +1231,8 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -1314,16 +1306,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" -dependencies = [ - "fallible-iterator", - "stable_deref_trait", -] - [[package]] name = "gimli" version = "0.27.2" @@ -1331,7 +1313,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 1.9.3", "stable_deref_trait", ] @@ -1356,20 +1338,9 @@ dependencies = [ [[package]] name = "goblin" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c955ab4e0ad8c843ea653a3d143048b87490d9be56bd7132a435c2407846ac8f" -dependencies = [ - "log", - "plain", - "scroll", -] - -[[package]] -name = "goblin" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572564d6cba7d09775202c8e7eebc4d534d5ae36578ab402fb21e182a0ac9505" +checksum = "0d6b4de4a8eb6c46a8c77e1d3be942cb9a8bf073c22374578e5ba4b08ed0ff68" dependencies = [ "log", "plain", @@ -1388,7 +1359,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util 0.7.7", @@ -1404,6 +1375,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "heck" version = "0.4.0" @@ -1596,7 +1573,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", ] [[package]] @@ -1975,19 +1962,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg", -] - -[[package]] -name = "miniz_oxide" -version = "0.5.4" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] @@ -2191,9 +2168,9 @@ dependencies = [ [[package]] name = "object" -version = "0.27.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c821014c18301591b89b843809ef953af9e3df0496c232d5c0611b0a52aac363" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" dependencies = [ "memchr", ] @@ -2388,8 +2365,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -2588,8 +2565,8 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -2650,9 +2627,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -2736,19 +2713,13 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.66", ] -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - [[package]] name = "rand" version = "0.7.3" @@ -3090,8 +3061,8 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "serde_derive_internals", "syn 1.0.103", ] @@ -3123,8 +3094,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -3159,22 +3130,22 @@ checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.13", + "proc-macro2 1.0.66", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -3183,18 +3154,18 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] [[package]] name = "serde_json" -version = "1.0.81" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ - "indexmap", + "indexmap 2.0.0", "itoa 1.0.1", "ryu", "serde", @@ -3229,7 +3200,7 @@ version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" dependencies = [ - "indexmap", + "indexmap 1.9.3", "itoa 1.0.1", "ryu", "serde", @@ -3344,8 +3315,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "410b26ed97440d90ced3e2488c868d56a86e2064f5d7d6f417909b286afe25e5" dependencies = [ "heck", - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -3454,8 +3425,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ "heck", - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "rustversion", "syn 1.0.103", ] @@ -3468,9 +3439,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "symbolic" -version = "10.2.0" +version = "12.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ac8ad1ebe348393d71802e8b0f5084c51fde21ad4c29ba8f8fb4d7ad6ed671" +checksum = "96f35e609846ed5924d00311809fdc22bfeebf0dfadfd33ec08bae7a8076aa7b" dependencies = [ "symbolic-common", "symbolic-debuginfo", @@ -3480,9 +3451,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "10.2.1" +version = "12.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" +checksum = "167a4ffd7c35c143fd1030aa3c2caf76ba42220bd5a6b5f4781896434723b8c3" dependencies = [ "debugid", "memmap2 0.5.10", @@ -3492,22 +3463,22 @@ dependencies = [ [[package]] name = "symbolic-debuginfo" -version = "10.2.0" +version = "12.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94766a96b5834eaf72f9cb99a5a45e63fa44f1084705b705d9d31bb6455434" +checksum = "214a188b70f3e82f56d068096fa9953cc6082b58a949b56629f5eaa30ceb574b" dependencies = [ - "bitvec", + "debugid", "dmsort", "elementtree", "elsa", "fallible-iterator", "flate2", - "gimli 0.26.2", - "goblin 0.6.0", + "gimli", + "goblin", "lazy_static", - "lazycell", "nom", "nom-supreme", + "once_cell", "parking_lot 0.12.1", "pdb-addr2line", "regex", @@ -3524,9 +3495,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "10.2.1" +version = "12.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" +checksum = "e378c50e80686c1c5c205674e1f86a2858bec3d2a7dfdd690331a8a19330f293" dependencies = [ "cc", "cpp_demangle", @@ -3537,11 +3508,14 @@ dependencies = [ [[package]] name = "symbolic-ppdb" -version = "10.2.1" +version = "12.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125fcd987182e46cd828416a9f2bdb7752c42081b33fa6d80a94afb1fdd4109b" +checksum = "5e1ce8734ea2f33b3b8f4a67d1fc58df295a0191f115eab60f9ac5cf7169129d" dependencies = [ - "indexmap", + "flate2", + "indexmap 1.9.3", + "serde", + "serde_json", "symbolic-common", "thiserror", "uuid", @@ -3550,11 +3524,11 @@ dependencies = [ [[package]] name = "symbolic-symcache" -version = "10.2.0" +version = "12.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789369a242bacbe89d2f4f6a364f54ea5df1dae774750eb30b335550b315749a" +checksum = "87f996fc4a38d97b9838a4de257c6d5359a2e9ccecc82d5a97b03e3b974f3082" dependencies = [ - "indexmap", + "indexmap 1.9.3", "symbolic-common", "symbolic-debuginfo", "thiserror", @@ -3579,28 +3553,22 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.13" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "unicode-ident", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "tempfile" version = "3.7.0" @@ -3638,9 +3606,9 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.13", + "proc-macro2 1.0.66", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -3703,9 +3671,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.13", + "proc-macro2 1.0.66", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -3755,7 +3723,7 @@ dependencies = [ "futures-io", "futures-sink", "futures-util", - "hashbrown", + "hashbrown 0.12.3", "pin-project-lite", "slab", "tokio", @@ -3786,8 +3754,8 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", ] @@ -3997,8 +3965,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", "wasm-bindgen-shared", ] @@ -4021,7 +3989,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ - "quote 1.0.26", + "quote 1.0.32", "wasm-bindgen-macro-support", ] @@ -4031,8 +3999,8 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -4046,11 +4014,12 @@ checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "wasmparser" -version = "0.94.0" +version = "0.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdac7e1d98d70913ae3b4923dd7419c8ea7bdfd4c44a240a0ba305d929b7f191" +checksum = "48134de3d7598219ab9eaf6b91b15d8e50d31da76b8519fe4ecfcec2cf35104b" dependencies = [ - "indexmap", + "indexmap 1.9.3", + "url", ] [[package]] @@ -4321,15 +4290,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "wyz" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" -dependencies = [ - "tap", -] - [[package]] name = "yaml-rust" version = "0.4.5" @@ -4347,9 +4307,9 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zip" -version = "0.6.3" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ "byteorder", "crc32fast", diff --git a/src/agent/coverage/Cargo.toml b/src/agent/coverage/Cargo.toml index ad87a12273..dfe7a94f3d 100644 --- a/src/agent/coverage/Cargo.toml +++ b/src/agent/coverage/Cargo.toml @@ -14,7 +14,7 @@ debuggable-module = { path = "../debuggable-module" } iced-x86 = "1.20" log = "0.4.17" regex = "1.9" -symbolic = { version = "10.1", features = [ +symbolic = { version = "12.3", features = [ "debuginfo", "demangle", "symcache", diff --git a/src/agent/debuggable-module/Cargo.toml b/src/agent/debuggable-module/Cargo.toml index b561841f35..915ad33bbe 100644 --- a/src/agent/debuggable-module/Cargo.toml +++ b/src/agent/debuggable-module/Cargo.toml @@ -13,7 +13,11 @@ iced-x86 = "1.20" log = "0.4.17" pdb = "0.8.0" regex = "1.9" -symbolic = { version = "10.1", features = ["debuginfo", "demangle", "symcache"] } +symbolic = { version = "12.3", features = [ + "debuginfo", + "demangle", + "symcache", +] } thiserror = "1.0" [dev-dependencies] diff --git a/src/agent/debugger/Cargo.toml b/src/agent/debugger/Cargo.toml index d52d429c12..a218b4905d 100644 --- a/src/agent/debugger/Cargo.toml +++ b/src/agent/debugger/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT" [dependencies] anyhow = "1.0" fnv = "1.0" -goblin = "0.5" +goblin = "0.6" iced-x86 = "1.20" log = "0.4" memmap2 = "0.7" @@ -18,4 +18,3 @@ win-util = { path = "../win-util" } [dependencies.windows] version = "0.48" - diff --git a/src/agent/onefuzz-task/src/local/common.rs b/src/agent/onefuzz-task/src/local/common.rs index b8cf6c06df..f8d7949e80 100644 --- a/src/agent/onefuzz-task/src/local/common.rs +++ b/src/agent/onefuzz-task/src/local/common.rs @@ -24,6 +24,7 @@ use crate::tasks::utils::parse_key_value; pub const SETUP_DIR: &str = "setup_dir"; pub const INPUTS_DIR: &str = "inputs_dir"; pub const CRASHES_DIR: &str = "crashes_dir"; +pub const CRASHDUMPS_DIR: &str = "crashdumps_dir"; pub const TARGET_WORKERS: &str = "target_workers"; pub const REPORTS_DIR: &str = "reports_dir"; pub const NO_REPRO_DIR: &str = "no_repro_dir"; diff --git a/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs b/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs index 74cb451189..80a083009b 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs @@ -6,8 +6,8 @@ use std::path::PathBuf; use crate::{ local::common::{ build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType, - SyncCountDirMonitor, UiEvent, CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV, - TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, + SyncCountDirMonitor, UiEvent, CHECK_FUZZER_HELP, CRASHDUMPS_DIR, CRASHES_DIR, INPUTS_DIR, + TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS, }, tasks::{ config::CommonConfig, @@ -27,6 +27,8 @@ pub fn build_fuzz_config( ) -> Result { let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)? .monitor_count(&event_sender)?; + let crashdumps = get_synced_dir(CRASHDUMPS_DIR, common.job_id, common.task_id, args)? + .monitor_count(&event_sender)?; let inputs = get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)? .monitor_count(&event_sender)?; @@ -49,6 +51,7 @@ pub fn build_fuzz_config( inputs, readonly_inputs, crashes, + crashdumps, target_exe, target_env, target_options, @@ -85,6 +88,10 @@ pub fn build_shared_args() -> Vec { .long(CRASHES_DIR) .required(true) .value_parser(value_parser!(PathBuf)), + Arg::new(CRASHDUMPS_DIR) + .long(CRASHDUMPS_DIR) + .required(true) + .value_parser(value_parser!(PathBuf)), Arg::new(TARGET_WORKERS) .long(TARGET_WORKERS) .value_parser(value_parser!(u64)), diff --git a/src/agent/onefuzz-task/src/local/template.rs b/src/agent/onefuzz-task/src/local/template.rs index f54e1f8eac..978dba74fa 100644 --- a/src/agent/onefuzz-task/src/local/template.rs +++ b/src/agent/onefuzz-task/src/local/template.rs @@ -51,6 +51,7 @@ struct LibFuzzer { inputs: PathBuf, readonly_inputs: Vec, crashes: PathBuf, + crashdumps: PathBuf, target_exe: PathBuf, target_env: HashMap, target_options: Vec, @@ -143,6 +144,7 @@ impl TaskConfig { inputs: context.to_monitored_sync_dir("inputs", &config.inputs)?, readonly_inputs: Some(ri?), crashes: context.to_monitored_sync_dir("crashes", &config.crashes)?, + crashdumps: context.to_monitored_sync_dir("crashdumps", &config.crashdumps)?, target_exe: config.target_exe.clone(), target_env: config.target_env.clone(), target_options: config.target_options.clone(), diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs index 35cfaa5952..ca51f5f5d7 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs @@ -13,12 +13,14 @@ use onefuzz::{ syncdir::{continuous_sync, SyncOperation::Pull, SyncedDir}, }; use onefuzz_telemetry::{ - Event::{new_coverage, new_result, runtime_stats}, + Event::{new_coverage, new_crashdump, new_result, runtime_stats}, EventData, }; use serde::Deserialize; use std::{ collections::HashMap, + ffi::{OsStr, OsString}, + fmt::Debug, path::{Path, PathBuf}, sync::Arc, }; @@ -72,6 +74,7 @@ pub struct Config { pub inputs: SyncedDir, pub readonly_inputs: Option>, pub crashes: SyncedDir, + pub crashdumps: SyncedDir, pub target_exe: PathBuf, pub target_env: HashMap, pub target_options: Vec, @@ -96,6 +99,7 @@ pub struct Config { pub struct LibFuzzerFuzzTask where L: LibFuzzerType, + Config: Debug, { config: Config, } @@ -103,6 +107,7 @@ where impl LibFuzzerFuzzTask where L: LibFuzzerType, + Config: Debug, { pub fn new(config: Config) -> Result { Ok(Self { config }) @@ -126,11 +131,19 @@ where let resync = self.continuous_sync_inputs(); let new_inputs = self.config.inputs.monitor_results(new_coverage, true); let new_crashes = self.config.crashes.monitor_results(new_result, true); + let new_crashdumps = self.config.crashdumps.monitor_results(new_crashdump, true); let (stats_sender, stats_receiver) = mpsc::unbounded_channel(); let report_stats = report_runtime_stats(stats_receiver, hb_client); let fuzzers = self.run_fuzzers(Some(&stats_sender)); - futures::try_join!(resync, new_inputs, new_crashes, fuzzers, report_stats)?; + futures::try_join!( + resync, + new_inputs, + new_crashes, + new_crashdumps, + fuzzers, + report_stats + )?; Ok(()) } @@ -235,8 +248,16 @@ where .for_each(|d| inputs.push(&d.local_path)); } + info!("config is: {:?}", self.config); + let fuzzer = L::from_config(&self.config).await?; let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs).await?; + + info!("child is: {:?}", running); + + #[cfg(target_os = "linux")] + let pid = running.id(); + let notify = Arc::new(Notify::new()); // Splitting borrow. @@ -247,19 +268,23 @@ where let mut stderr = BufReader::new(stderr); let mut libfuzzer_output: ArrayDeque<_, LOGS_BUFFER_SIZE, Wrapping> = ArrayDeque::new(); - loop { + { let mut buf = vec![]; - let bytes_read = stderr.read_until(b'\n', &mut buf).await?; - if bytes_read == 0 && buf.is_empty() { - break; - } - let line = String::from_utf8_lossy(&buf).to_string(); - if let Some(stats_sender) = stats_sender { - if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, &line) { - error!("could not parse fuzzing interation update: {}", err); + loop { + buf.clear(); + let bytes_read = stderr.read_until(b'\n', &mut buf).await?; + if bytes_read == 0 && buf.is_empty() { + break; + } + let line = String::from_utf8_lossy(&buf).to_string(); + if let Some(stats_sender) = stats_sender { + if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, &line) + { + error!("could not parse fuzzing interation update: {}", err); + } } + libfuzzer_output.push_back(line); } - libfuzzer_output.push_back(line); } let exit_status = running.wait().await; @@ -267,8 +292,20 @@ where let exit_status: ExitStatus = exit_status?.into(); + info!( + "fuzzer exited, here are the last {} lines of stderr:", + libfuzzer_output.len() + ); + info!("------------------------"); + for line in libfuzzer_output.iter() { + info!("{}", line.trim_end_matches('\n')); + } + info!("------------------------"); + let files = list_files(crash_dir.path()).await?; + info!("found {} crashes", files.len()); + // If the target exits, crashes are required unless // 1. Exited cleanly (happens with -runs=N) // 2. expect_crash_on_failure is disabled @@ -291,10 +328,21 @@ where } } - for file in &files { + // name the dumpfile after the crash file (if one) + // otherwise don't rename it + let dump_file_name: Option = if files.len() == 1 { + files + .first() + .and_then(|f| f.file_name().map(OsStr::to_os_string)) + } else { + None + }; + + // move crashing inputs to output directory + for file in files { if let Some(filename) = file.file_name() { let dest = self.config.crashes.local_path.join(filename); - if let Err(e) = tokio::fs::rename(file.clone(), dest.clone()).await { + if let Err(e) = tokio::fs::rename(file, dest.clone()).await { if !dest.exists() { bail!(e) } @@ -302,17 +350,83 @@ where } } + // check for core dumps on Linux: + // note that collecting the dumps must be enabled by the template + #[cfg(target_os = "linux")] + if let Some(pid) = pid { + // expect crash dump to exist in CWD + let filename = format!("core.{pid}"); + let dest_filename = dump_file_name.as_deref().unwrap_or(OsStr::new(&filename)); + let dest_path = self.config.crashdumps.local_path.join(dest_filename); + match tokio::fs::rename(&filename, &dest_path).await { + Ok(()) => { + info!( + "moved crash dump {} to output directory: {}", + filename, + dest_path.display() + ); + } + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + // okay, no crash dump found + info!("no crash dump found with name: {}", filename); + } else { + return Err(e).context("moving crash dump to output directory"); + } + } + } + } else { + warn!("no PID found for libfuzzer process"); + } + + // check for crash dumps on Windows: + #[cfg(target_os = "windows")] + { + let dumpfile_extension = Some(std::ffi::OsStr::new("dmp")); + + let mut working_dir = tokio::fs::read_dir(".").await?; + let mut found_dump = false; + while let Some(next) = working_dir.next_entry().await? { + if next.path().extension() == dumpfile_extension { + // Windows dumps get a fixed filename so we will generate a random one, + // if there's no valid target crash name: + let dest_filename = + dump_file_name.unwrap_or_else(|| uuid::Uuid::new_v4().to_string().into()); + let dest_path = self.config.crashdumps.local_path.join(&dest_filename); + tokio::fs::rename(next.path(), &dest_path) + .await + .context("moving crash dump to output directory")?; + info!( + "moved crash dump {} to output directory: {}", + next.path().display(), + dest_path.display() + ); + found_dump = true; + break; + } + } + + if !found_dump { + info!("no crash dump found with extension .dmp"); + } + } + Ok(()) } async fn init_directories(&self) -> Result<()> { + // input directories (init_pull): self.config.inputs.init_pull().await?; - self.config.crashes.init().await?; if let Some(readonly_inputs) = &self.config.readonly_inputs { for dir in readonly_inputs { dir.init_pull().await?; } } + + // output directories (init): + self.config.crashes.init().await?; + self.config.crashdumps.init().await?; + Ok(()) } diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs index b66cff2060..f1a750ccf7 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs @@ -20,7 +20,7 @@ use onefuzz::{ SyncedDir, }, }; -use onefuzz_telemetry::Event::{new_coverage, new_result}; +use onefuzz_telemetry::Event::{new_coverage, new_crashdump, new_result}; use serde::Deserialize; use std::{ collections::HashMap, @@ -41,6 +41,7 @@ use futures::TryFutureExt; pub struct SupervisorConfig { pub inputs: SyncedDir, pub crashes: SyncedDir, + pub crashdumps: SyncedDir, pub supervisor_exe: String, pub supervisor_env: HashMap, pub supervisor_options: Vec, @@ -80,6 +81,14 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { crashes.init().await?; let monitor_crashes = crashes.monitor_results(new_result, false); + // setup crashdumps + let crashdumps = SyncedDir { + local_path: runtime_dir.path().join("crashdumps"), + remote_path: config.crashdumps.remote_path.clone(), + }; + crashdumps.init().await?; + let monitor_crashdumps = crashdumps.monitor_results(new_crashdump, false); + // setup coverage if let Some(coverage) = &config.coverage { coverage.init_pull().await?; @@ -139,6 +148,7 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { &runtime_dir.path(), &config, &crashes, + &crashdumps, &inputs, reports_dir.path().to_path_buf(), ) @@ -170,6 +180,7 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { monitor_supervisor.map_err(|e| e.context("Failure in monitor_supervisor")), monitor_stats.map_err(|e| e.context("Failure in monitor_stats")), monitor_crashes.map_err(|e| e.context("Failure in monitor_crashes")), + monitor_crashdumps.map_err(|e| e.context("Failure in monitor_crashdumps")), monitor_inputs.map_err(|e| e.context("Failure in monitor_inputs")), inputs_sync_task.map_err(|e| e.context("Failure in continuous_sync_task")), monitor_reports_future.map_err(|e| e.context("Failure in monitor_reports_future")), @@ -206,6 +217,7 @@ async fn start_supervisor( runtime_dir: impl AsRef, config: &SupervisorConfig, crashes: &SyncedDir, + crashdumps: &SyncedDir, inputs: &SyncedDir, reports_dir: PathBuf, ) -> Result { @@ -221,6 +233,7 @@ async fn start_supervisor( .supervisor_options(&config.supervisor_options) .runtime_dir(&runtime_dir) .crashes(&crashes.local_path) + .crashdumps(&crashdumps.local_path) .input_corpus(&inputs.local_path) .reports_dir(reports_dir) .setup_dir(&config.common.setup_dir) @@ -245,15 +258,15 @@ async fn start_supervisor( .set_optional_ref(&config.target_options, |expand, target_options| { expand.target_options(target_options) }) - .set_optional_ref(&config.common.microsoft_telemetry_key, |tester, key| { - tester.microsoft_telemetry_key(key) + .set_optional_ref(&config.common.microsoft_telemetry_key, |expand, key| { + expand.microsoft_telemetry_key(key) }) - .set_optional_ref(&config.common.instance_telemetry_key, |tester, key| { - tester.instance_telemetry_key(key) + .set_optional_ref(&config.common.instance_telemetry_key, |expand, key| { + expand.instance_telemetry_key(key) }) .set_optional_ref( &config.crashes.remote_path.clone().and_then(|u| u.account()), - |tester, account| tester.crashes_account(account), + |expand, account| expand.crashes_account(account), ) .set_optional_ref( &config @@ -261,7 +274,7 @@ async fn start_supervisor( .remote_path .clone() .and_then(|u| u.container()), - |tester, container| tester.crashes_container(container), + |expand, container| expand.crashes_container(container), ); let supervisor_path = expand.evaluate_value(&config.supervisor_exe)?; @@ -340,7 +353,6 @@ mod tests { let fault_dir_temp = tempfile::tempdir().unwrap(); let crashes_local = tempfile::tempdir().unwrap().path().into(); - let corpus_dir_local = tempfile::tempdir().unwrap().path().into(); let crashes = SyncedDir { local_path: crashes_local, remote_path: Some( @@ -348,6 +360,17 @@ mod tests { ), }; + let crashdumps_dir_temp = tempfile::tempdir().unwrap(); + let crashdumps_local = tempfile::tempdir().unwrap().path().into(); + let crashdumps = SyncedDir { + local_path: crashdumps_local, + remote_path: Some( + BlobContainerUrl::parse(Url::from_directory_path(crashdumps_dir_temp).unwrap()) + .unwrap(), + ), + }; + + let corpus_dir_local = tempfile::tempdir().unwrap().path().into(); let corpus_dir_temp = tempfile::tempdir().unwrap(); let corpus_dir = SyncedDir { local_path: corpus_dir_local, @@ -387,6 +410,7 @@ mod tests { target_options, inputs: corpus_dir.clone(), crashes: crashes.clone(), + crashdumps: crashdumps.clone(), tools: None, wait_for_files: None, stats_file: None, @@ -419,9 +443,16 @@ mod tests { }, }; - let process = start_supervisor(runtime_dir, &config, &crashes, &corpus_dir, reports_dir) - .await - .unwrap(); + let process = start_supervisor( + runtime_dir, + &config, + &crashes, + &crashdumps, + &corpus_dir, + reports_dir, + ) + .await + .unwrap(); let notify = Notify::new(); let _fuzzing_monitor = diff --git a/src/agent/onefuzz-telemetry/src/lib.rs b/src/agent/onefuzz-telemetry/src/lib.rs index e8ad34f345..f08b722695 100644 --- a/src/agent/onefuzz-telemetry/src/lib.rs +++ b/src/agent/onefuzz-telemetry/src/lib.rs @@ -77,6 +77,7 @@ pub enum Event { coverage_data, coverage_failed, new_result, + new_crashdump, new_coverage, runtime_stats, new_report, @@ -93,6 +94,7 @@ impl Event { Self::coverage_data => "coverage_data", Self::coverage_failed => "coverage_failed", Self::new_coverage => "new_coverage", + Self::new_crashdump => "new_crashdump", Self::new_result => "new_result", Self::runtime_stats => "runtime_stats", Self::new_report => "new_report", diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index db858810a4..93587a6b58 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -24,6 +24,7 @@ type MappingFn<'a> = Box) -> Result> + Send pub enum PlaceHolder { Input, Crashes, + Crashdumps, InputCorpus, GeneratedInputs, TargetExe, @@ -59,6 +60,7 @@ impl PlaceHolder { match self { Self::Input => "{input}", Self::Crashes => "{crashes}", + Self::Crashdumps => "{crashdumps}", Self::InputCorpus => "{input_corpus}", Self::GeneratedInputs => "{generated_inputs}", Self::TargetExe => "{target_exe}", @@ -234,6 +236,12 @@ impl<'a> Expand<'a> { self.set_value(PlaceHolder::Crashes, ExpandedValue::Path(path)) } + pub fn crashdumps(self, arg: impl AsRef) -> Self { + let arg = arg.as_ref(); + let path = String::from(arg.to_string_lossy()); + self.set_value(PlaceHolder::Crashdumps, ExpandedValue::Path(path)) + } + pub fn input_path(self, arg: impl AsRef) -> Self { let arg = arg.as_ref(); let path = String::from(arg.to_string_lossy()); diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 26ec0f0bfd..6968192642 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1087,11 +1087,10 @@ def create( if tags is None: tags = {} - containers_submit = [] - for container_type, container in containers: - containers_submit.append( - models.TaskContainers(name=container, type=container_type) - ) + containers_submit = [ + models.TaskContainers(name=container, type=container_type) + for container_type, container in containers + ] config = models.TaskConfig( containers=containers_submit, @@ -1210,6 +1209,7 @@ def delete( ) -> None: SAFE_TO_REMOVE = [ enums.ContainerType.crashes, + enums.ContainerType.crashdumps, enums.ContainerType.setup, enums.ContainerType.inputs, enums.ContainerType.reports, diff --git a/src/cli/onefuzz/templates/afl.py b/src/cli/onefuzz/templates/afl.py index 484fca1713..d3019a19cf 100644 --- a/src/cli/onefuzz/templates/afl.py +++ b/src/cli/onefuzz/templates/afl.py @@ -45,6 +45,7 @@ def basic( supervisor_env: Optional[Dict[str, str]] = None, supervisor_input_marker: str = "@@", tags: Optional[Dict[str, str]] = None, + target_env: Optional[Dict[str, str]] = None, wait_for_running: bool = False, wait_for_files: Optional[List[ContainerType]] = None, afl_container: Optional[Container] = None, @@ -162,6 +163,7 @@ def basic( stats_format=StatsFormat.AFL, task_wait_for_files=ContainerType.inputs, tags=helper.tags, + target_env=target_env, debug=debug, ensemble_sync_delay=ensemble_sync_delay, ) diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index 2dd1891e79..7716cfefed 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -132,6 +132,7 @@ def _create_tasks( fuzzer_containers = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.crashes, containers[ContainerType.crashes]), + (ContainerType.crashdumps, containers[ContainerType.crashdumps]), (ContainerType.inputs, containers[ContainerType.inputs]), ] @@ -416,6 +417,7 @@ def basic( ContainerType.setup, ContainerType.inputs, ContainerType.crashes, + ContainerType.crashdumps, ContainerType.reports, ContainerType.unique_reports, ContainerType.unique_inputs, @@ -726,6 +728,7 @@ def dotnet( ContainerType.setup, ContainerType.inputs, ContainerType.crashes, + ContainerType.crashdumps, ContainerType.coverage, ContainerType.reports, ContainerType.unique_reports, @@ -753,6 +756,7 @@ def dotnet( fuzzer_containers = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.crashes, containers[ContainerType.crashes]), + (ContainerType.crashdumps, containers[ContainerType.crashdumps]), (ContainerType.inputs, containers[ContainerType.inputs]), (ContainerType.tools, fuzzer_tools_container), ] @@ -960,6 +964,7 @@ def qemu_user( ContainerType.setup, ContainerType.inputs, ContainerType.crashes, + ContainerType.crashdumps, ContainerType.reports, ContainerType.unique_reports, ContainerType.no_repro, @@ -978,6 +983,7 @@ def qemu_user( fuzzer_containers = [ (ContainerType.setup, helper.containers[ContainerType.setup]), (ContainerType.crashes, helper.containers[ContainerType.crashes]), + (ContainerType.crashdumps, helper.containers[ContainerType.crashdumps]), (ContainerType.inputs, helper.containers[ContainerType.inputs]), ] diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index 0a653144c5..057404ceff 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -87,6 +87,7 @@ class Integration(BaseModel): target_class: Optional[str] target_method: Optional[str] setup_dir: Optional[str] + target_env: Optional[Dict[str, str]] TARGETS: Dict[str, Integration] = { @@ -106,12 +107,21 @@ class Integration(BaseModel): ContainerType.unique_reports: 1, ContainerType.coverage: 1, ContainerType.inputs: 2, + # TODO: crashdumps are intermittently not captured + # during integration tests on Linux. This requires more + # investigation before we can fully enable this test. + # ContainerType.crashdumps: 1, ContainerType.extra_output: 1, }, reboot_after_setup=True, inject_fake_regression=True, + target_env={ + # same TODO + # "ASAN_OPTIONS": "disable_coredump=0:abort_on_error=1:unmap_shadow_on_exit=1" + }, fuzzing_target_options=[ "--test:{extra_setup_dir}", + "--only_asan_failures", "--write_test_file={extra_output_dir}/test.txt", ], ), @@ -213,11 +223,15 @@ class Integration(BaseModel): ContainerType.inputs: 2, ContainerType.unique_reports: 1, ContainerType.coverage: 1, + ContainerType.crashdumps: 1, ContainerType.extra_output: 1, }, inject_fake_regression=True, + target_env={"ASAN_SAVE_DUMPS": "my_dump.dmp"}, + # we should set unmap_shadow_on_exit=1 but it fails on Windows at the moment fuzzing_target_options=[ "--test:{extra_setup_dir}", + "--only_asan_failures", "--write_test_file={extra_output_dir}/test.txt", ], ), @@ -631,6 +645,7 @@ def build_job( fuzzing_target_options=config.fuzzing_target_options, extra_setup_container=Container(extra_setup_container.name), extra_output_container=Container(extra_output_container.name), + target_env=config.target_env, ) elif config.template == TemplateType.libfuzzer_dotnet: if setup is None: @@ -653,6 +668,7 @@ def build_job( fuzzing_target_options=config.target_options, target_class=config.target_class, target_method=config.target_method, + target_env=config.target_env, ) elif config.template == TemplateType.libfuzzer_qemu_user: return self.of.template.libfuzzer.qemu_user( @@ -665,6 +681,7 @@ def build_job( duration=duration, vm_count=1, target_options=config.target_options, + target_env=config.target_env, ) elif config.template == TemplateType.radamsa: return self.of.template.radamsa.basic( @@ -679,6 +696,7 @@ def build_job( disable_check_debugger=config.disable_check_debugger or False, duration=duration, vm_count=1, + target_env=config.target_env, ) elif config.template == TemplateType.afl: return self.of.template.afl.basic( @@ -692,6 +710,7 @@ def build_job( duration=duration, vm_count=1, target_options=config.target_options, + target_env=config.target_env, ) else: raise NotImplementedError @@ -798,14 +817,18 @@ def check_jobs_impl() -> Tuple[bool, str, bool]: return (True, "timed out while checking jobs", False) for job_id in check_containers: + job_name = jobs[job_id].config.name finished_containers: Set[Container] = set() for container_name, container_impl in check_containers[job_id].items(): - container_client, count = container_impl - if len(container_client.list_blobs()) >= count: + container_client, required_count = container_impl + found_count = len(container_client.list_blobs()) + if found_count >= required_count: clear() self.logger.info( - "found files for %s - %s", - jobs[job_id].config.name, + "found %d files (needed %d) for %s - %s", + found_count, + required_count, + job_name, container_name, ) finished_containers.add(container_name) @@ -813,6 +836,12 @@ def check_jobs_impl() -> Tuple[bool, str, bool]: for container_name in finished_containers: del check_containers[job_id][container_name] + to_check = check_containers[job_id].keys() + if len(to_check) > 0: + self.logger.info( + "%s - still waiting for %s", job_name, ", ".join(to_check) + ) + scalesets = self.of.scalesets.list() for job_id in job_tasks: finished_tasks: Set[UUID] = set() diff --git a/src/integration-tests/libfuzzer/Makefile b/src/integration-tests/libfuzzer/Makefile index 3735cfcb57..648d5b0cda 100644 --- a/src/integration-tests/libfuzzer/Makefile +++ b/src/integration-tests/libfuzzer/Makefile @@ -1,6 +1,10 @@ -CC=clang - -CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address +ifeq ($(OS),Windows_NT) + CFLAGS=/O2 /Zi /fsanitize=fuzzer /fsanitize=address + CC=cl +else + CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address + CC=clang +endif all: fuzz.exe diff --git a/src/integration-tests/libfuzzer/simple.c b/src/integration-tests/libfuzzer/simple.c index d776f5c652..209f992df8 100644 --- a/src/integration-tests/libfuzzer/simple.c +++ b/src/integration-tests/libfuzzer/simple.c @@ -7,93 +7,151 @@ #include #include -// allow an argument --write_test_file=xxx.txt to be set -// which is useful for exercising some OneFuzz features in integration tests -int LLVMFuzzerInitialize(int *argc, char ***argv) { - const int num_args = *argc; - char** args = *argv; +bool only_asan = false; - for (int i = 0; i < num_args; ++i) { - // look for argument starting with --write_test_file= - const char* arg_name = "--write_test_file="; - if (strncmp(args[i], arg_name, strlen(arg_name)) == 0) { - // extract filename part - const char* file_name = args[i] + strlen(arg_name); - // write file - FILE* output = fopen(file_name, "a"); - if (!output) { - perror("failed to open file"); - return -1; - } +int LLVMFuzzerInitialize(int *argc, char ***argv) +{ + const int num_args = *argc; + char **args = *argv; - fputs("Hello from simple fuzzer\n", output); - fclose(output); - break; + for (int i = 0; i < num_args; ++i) + { + // allow an argument --write_test_file=xxx.txt to be set + // which is useful for exercising some OneFuzz features in integration tests + const char *test_file_arg = "--write_test_file="; + // look for argument starting with --write_test_file= + if (strncmp(args[i], test_file_arg, strlen(test_file_arg)) == 0) + { + // extract filename part + const char *file_name = args[i] + strlen(test_file_arg); + // write file + FILE *output = fopen(file_name, "a"); + if (!output) + { + perror("failed to open file"); + return -1; + } + + fputs("Hello from simple fuzzer\n", output); + fclose(output); + } + + // an argument to only allow generating ASAN failures + const char *asan_only_arg = "--only_asan_failures"; + if (strcmp(args[i], asan_only_arg) == 0) + { + only_asan = true; + } } - } - return 0; + return 0; } -int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) { - int cnt = 0; +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) +{ + int cnt = 0; - if (len < 4) { - return 0; - } + if (len < 4) + { + return 0; + } - if (data[0] == 'x') { cnt++; } - if (data[1] == 'y') { cnt++; } - if (data[2] == 'z') { cnt++; } + if (data[0] == 'x') + { + cnt++; + } + if (data[1] == 'y') + { + cnt++; + } + if (data[2] == 'z') + { + cnt++; + } - if (cnt >= 3) { - switch (data[3]) { - case '0': { - // segv - int *p = NULL; *p = 123; - break; - } - case '1': { - // stack-buffer-underflow - int* p = &cnt - 32; for (int i = 0; i < 32; i++) { *(p + i) = 0; } - break; - } - case '2': { - // stack-buffer-overflow - int* p = &cnt + 32; for (int i = 0; i < 32; i++) { *(p - i) = 0; } - break; - } - case '3': { - // bad-free - int *p = &cnt; free(p); - break; - } - case '4': { - // double-free - int* p = (int *) malloc(sizeof(int)); free(p); free(p); - break; - } - case '5': { - // heap-use-after-free - int* p = (int *) malloc(sizeof(int)); free(p); *p = 123; - break; - } - case '6': { - // heap-buffer-overflow - int* p = (int *) malloc(8 * sizeof(int)); for (int i = 0; i < 32; i++) { *(p + i) = 0; } - break; - } - case '7': { - // fpe - int x = 0; int y = 123 / x; - break; - } - case '8': { - abort(); - break; - } + if (cnt >= 3) + { + switch (data[3]) + { + case '0': + { + // segv + int *p = NULL; + *p = 123; + break; + } + case '1': + { + // stack-buffer-underflow + int *p = &cnt - 32; + for (int i = 0; i < 32; i++) + { + *(p + i) = 0; + } + break; + } + case '2': + { + // stack-buffer-overflow + int *p = &cnt + 32; + for (int i = 0; i < 32; i++) + { + *(p - i) = 0; + } + break; + } + case '3': + { + // bad-free + int *p = &cnt; + free(p); + break; + } + case '4': + { + // double-free + int *p = (int *)malloc(sizeof(int)); + free(p); + free(p); + break; + } + case '5': + { + // heap-use-after-free + int *p = (int *)malloc(sizeof(int)); + free(p); + *p = 123; + break; + } + case '6': + { + // heap-buffer-overflow + int *p = (int *)malloc(8 * sizeof(int)); + for (int i = 0; i < 32; i++) + { + *(p + i) = 0; + } + break; + } + case '7': + { + // fpe + int x = 0; + int y = 123 / x; + break; + } + case '8': + { + if (only_asan) + { + break; + } + + abort(); + break; + } + } } - } - return 0; + return 0; } diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index 949ace759d..cf09b01824 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -219,6 +219,7 @@ class ContainerType(Enum): coverage = "coverage" crashes = "crashes" inputs = "inputs" + crashdumps = "crashdumps" no_repro = "no_repro" readonly_inputs = "readonly_inputs" reports = "reports" @@ -237,6 +238,7 @@ def reset_defaults(cls) -> List["ContainerType"]: cls.analysis, cls.coverage, cls.crashes, + cls.crashdumps, cls.inputs, cls.no_repro, cls.readonly_inputs, diff --git a/src/runtime-tools/linux/run.sh b/src/runtime-tools/linux/run.sh index 9c2e8e8038..897c0eac80 100755 --- a/src/runtime-tools/linux/run.sh +++ b/src/runtime-tools/linux/run.sh @@ -16,21 +16,24 @@ logger "onefuzz: starting up onefuzz" #check if we are running in docker if [ -f /.dockerenv ]; then - echo "Running in docker: - to optimize the experience make sure the host os is setup properly. with the following command - # use core files, not external crash handler - echo core | sudo tee /proc/sys/kernel/core_pattern - # disable ASLR - echo 0 | sudo tee /proc/sys/kernel/randomize_va_space - # set core dumping to default behavior - echo 1 | sudo tee /proc/sys/fs/suid_dumpable" + echo "Running in docker: to optimize the experience make sure the host OS is setup properly, use the following commands: + # 1) use core files, not external crash handler + # 2) suffix core with PID: will be 'core.XXXX' + # 3) disable ASLR + # 4) set core dumping to default behavior + sudo sysctl -w 'kernel.core_pattern=core' 'kernel.core_uses_pid=1' 'kernel.randomize_va_space=0' 'fs.suid_dumpable=1' + + # unlimit core files + ulimit -c unlimited" else - # use core files, not external crash handler - echo core | sudo tee /proc/sys/kernel/core_pattern - # disable ASLR - echo 0 | sudo tee /proc/sys/kernel/randomize_va_space - # set core dumping to default behavior - echo 1 | sudo tee /proc/sys/fs/suid_dumpable + # 1) use core files, not external crash handler + # 2) suffix core with PID: will be 'core.XXXX' + # 3) disable ASLR + # 4) set core dumping to default behavior + sudo sysctl -w 'kernel.core_pattern=core' 'kernel.core_uses_pid=1' 'kernel.randomize_va_space=0' 'fs.suid_dumpable=1' + + # unlimit core files + ulimit -c unlimited fi cd /onefuzz