Merge branch 'main' into mgreisen/archivemsg

.devcontainer/devcontainer.json

@@ -13,6 +13,7 @@
 					"**/target/**": true
 				},
 				"lldb.executable": "/usr/bin/lldb",
+				"dotnet.server.useOmnisharp": true,
 				"omnisharp.enableEditorConfigSupport": true,
 				"omnisharp.enableRoslynAnalyzers": true,
 				"python.defaultInterpreterPath": "/workspaces/onefuzz/src/venv/bin/python",
@@ -48,4 +49,4 @@
 	"features": {
 		"ghcr.io/devcontainers/features/azure-cli:1": {}
 	}
-}
+}

.github/workflows/ci.yml

@@ -375,6 +375,10 @@ jobs:
         with:
           # use global.json to install the correct version
           global-json-file: global.json
+      - name: install llVM
+        run: |
+          choco install llvm --version 16.0.6
+        shell: powershell
       - run: src/ci/dotnet-fuzzing-tools.ps1
         shell: pwsh
       - uses: actions/upload-artifact@v3
@@ -538,9 +542,11 @@ jobs:
 
           mkdir -p artifacts/linux-libfuzzer
           mkdir -p artifacts/linux-libfuzzer-with-options
+          mkdir -p artifacts/mariner-libfuzzer
           (cd libfuzzer ; make )
           cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer
           cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer-with-options
+          cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/mariner-libfuzzer
 
           mkdir -p artifacts/linux-libfuzzer-regression
           (cd libfuzzer-regression ; make )

CHANGELOG.md

@@ -7,6 +7,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 8.7.0
+
+### Added
+
+* Agent: Added a snapshot-based test to coverage implementation [#3368](https://github.com/microsoft/onefuzz/pull/3368)
+* Agent/CLI/Service: Added ability to capture crash dumps from libfuzzer, when provided [#2793](https://github.com/microsoft/onefuzz/pull/2793) [#3409](https://github.com/microsoft/onefuzz/pull/3409)
+* CLI/Service: Implemented `--with_tasks ` option for `onefuzz jobs get` command to expand the task information [#3343](https://github.com/microsoft/onefuzz/pull/3343)
+
+### Changed
+
+* Agent: Migrated all the task types to the template model [#3397](https://github.com/microsoft/onefuzz/pull/3307)
+* Agent: Removed `srcview` code from OneFuzz since it is not currently utilized [#3376](https://github.com/microsoft/onefuzz/pull/3376)
+* Agent: Updated default windows VM image to windows 11 [#3374](https://github.com/microsoft/onefuzz/pull/3374)
+* Agent: Migrated `winapi` to `windows-rs`, the newer Microsoft supported version of the Windows API bindings for Rust [#3050](https://github.com/microsoft/onefuzz/pull/3050)
+* Deployment: Updated the default deployment option for `EnableWorkItemCreation` feature flag to be enabled [#3387](https://github.com/microsoft/onefuzz/pull/3387)
+
+### Fixed
+
+* Agent: Deserialize the coverage files directly into the output files [#3410](https://github.com/microsoft/onefuzz/pull/3410)
+* Agent/Deployment/Service: Bumped several C#, Python, and Rust dependencies as well as the Rust edition across all Rust crates [#3396](https://github.com/microsoft/onefuzz/pull/3396), [#3161](https://github.com/microsoft/onefuzz/pull/3161), [#3346](https://github.com/microsoft/onefuzz/pull/3346), [#3391](https://github.com/microsoft/onefuzz/pull/3391), [#2870](https://github.com/microsoft/onefuzz/pull/2870), [#3392](https://github.com/microsoft/onefuzz/pull/3392), [#3402](https://github.com/microsoft/onefuzz/pull/3402)
+* Agent: Fixed a bug in agent `DirectoryMonitor` by adding error tolerance when attempting to fetch metadata for `CreateKind::Any` or `CreateKind::Other` events [#3393](https://github.com/microsoft/onefuzz/pull/3393)
+* Service: Fixed tag shadowing in logging by giving precedence to the tags produced by log messages over the tags added prior to the call, when the tag names clashed [#3388](https://github.com/microsoft/onefuzz/pull/3388)
+
 ## 8.6.3
 
 ### Fixed

CURRENT_VERSION

@@ -1 +1 @@
-8.6.3
+8.7.0

docs/supported-platforms.md

@@ -4,18 +4,18 @@ OneFuzz is cross-platform, and the actively-supported platforms vary by componen
 
 ### CLI
 
-We continuously test the CLI on Windows 10 Pro and Ubuntu 20.04 LTS, both on the
-x64 architecture.  The CLI client is written in Python 3, and targets Python 3.7
-and up.  We distribute a self-contained executable CLI build for Windows which 
+We continuously test the CLI on Windows 11 Pro and Ubuntu 20.04 LTS, both on the
+x64 architecture. The CLI client is written in Python 3, and targets Python 3.7
+and up. We distribute a self-contained executable CLI build for Windows which
 bundles a Python interpreter.
 
 ### Virtual Machine Scale Sets
 
 OneFuzz deploys targets into Azure Virtual Machine Scale Sets for fuzzing (and
-supporting tasks).  OneFuzz permits arbitrary choice of VM SKU and OS Image,
-including custom images.  We continuously test on Window 10 Pro x64 (using the 
-Azure OS image URN `MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest`)
-and Ubuntu 20.04 LTS x64 (using the Azure OS image URN 
+supporting tasks). OneFuzz permits arbitrary choice of VM SKU and OS Image,
+including custom images. We continuously test on Window 11 Pro x64 (using the
+Azure OS image URN `MicrosoftWindowsDesktop:Windows-11:win11-22h2-pro:latest`)
+and Ubuntu 20.04 LTS x64 (using the Azure OS image URN
 `Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest`).
 
 ### LibFuzzer Compilation

docs/webhook_events.md

@@ -721,7 +721,7 @@ If webhook is set to have Event Grid message format then the payload will look a
             "00000000-0000-0000-0000-000000000000"
         ],
         "default_linux_vm_image": "Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest",
-        "default_windows_vm_image": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
+        "default_windows_vm_image": "MicrosoftWindowsDesktop:Windows-11:win11-22h2-pro:latest",
         "network_config": {
             "address_space": "10.0.0.0/8",
             "subnet": "10.0.0.0/16"
@@ -868,7 +868,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "type": "string"
                 },
                 "default_windows_vm_image": {
-                    "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
+                    "default": "MicrosoftWindowsDesktop:Windows-11:win11-22h2-pro:latest",
                     "title": "Default Windows Vm Image",
                     "type": "string"
                 },
@@ -6181,7 +6181,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "type": "string"
                 },
                 "default_windows_vm_image": {
-                    "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
+                    "default": "MicrosoftWindowsDesktop:Windows-11:win11-22h2-pro:latest",
                     "title": "Default Windows Vm Image",
                     "type": "string"
                 },

src/ApiService/ApiService/Functions/NotificationsTest.cs

@@ -24,6 +24,11 @@ public async Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.An
         }
 
         var notificationTest = request.OkV;
+        var validConfig = await notificationTest.Notification.Config.Validate();
+        if (!validConfig.IsOk) {
+            return await _context.RequestHandling.NotOk(req, validConfig.ErrorV, context: "notification test");
+        }
+
         var result = await _context.NotificationOperations.TriggerNotification(notificationTest.Notification.Container, notificationTest.Notification,
             notificationTest.Report, isLastRetryAttempt: true);
         var response = req.CreateResponse(HttpStatusCode.OK);

src/ApiService/ApiService/Functions/QueueJobResult.cs

@@ -0,0 +1,60 @@
+using System.Text.Json;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Extensions.Logging;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
+namespace Microsoft.OneFuzz.Service.Functions;
+
+
+public class QueueJobResult {
+    private readonly ILogger _log;
+    private readonly IOnefuzzContext _context;
+
+    public QueueJobResult(ILogger<QueueJobResult> logTracer, IOnefuzzContext context) {
+        _log = logTracer;
+        _context = context;
+    }
+
+    [Function("QueueJobResult")]
+    public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJobsStorage")] string msg) {
+
+        var _tasks = _context.TaskOperations;
+        var _jobs = _context.JobOperations;
+
+        _log.LogInformation("job result: {msg}", msg);
+        var jr = JsonSerializer.Deserialize<TaskJobResultEntry>(msg, EntityConverter.GetJsonSerializerOptions()).EnsureNotNull($"wrong data {msg}");
+
+        var task = await _tasks.GetByTaskId(jr.TaskId);
+        if (task == null) {
+            _log.LogWarning("invalid {TaskId}", jr.TaskId);
+            return;
+        }
+
+        var job = await _jobs.Get(task.JobId);
+        if (job == null) {
+            _log.LogWarning("invalid {JobId}", task.JobId);
+            return;
+        }
+
+        JobResultData? data = jr.Data;
+        if (data == null) {
+            _log.LogWarning($"job result data is empty, throwing out: {jr}");
+            return;
+        }
+
+        var jobResultType = data.Type;
+        _log.LogInformation($"job result data type: {jobResultType}");
+
+        Dictionary<string, double> value;
+        if (jr.Value.Count > 0) {
+            value = jr.Value;
+        } else {
+            _log.LogWarning($"job result data is empty, throwing out: {jr}");
+            return;
+        }
+
+        var jobResult = await _context.JobResultOperations.CreateOrUpdate(job.JobId, jobResultType, value);
+        if (!jobResult.IsOk) {
+            _log.LogError("failed to create or update with job result {JobId}", job.JobId);
+        }
+    }
+}

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -48,6 +48,7 @@ public enum ErrorCode {
     ADO_VALIDATION_UNEXPECTED_ERROR = 491,
     ADO_VALIDATION_MISSING_PAT_SCOPES = 492,
     ADO_WORKITEM_PROCESSING_DISABLED = 494,
+    ADO_VALIDATION_INVALID_PATH = 495,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -33,6 +33,19 @@ public enum HeartbeatType {
     TaskAlive,
 }
 
+[SkipRename]
+public enum JobResultType {
+    NewCrashingInput,
+    NoReproCrashingInput,
+    NewReport,
+    NewUniqueReport,
+    NewRegressionReport,
+    NewCoverage,
+    NewCrashDump,
+    CoverageData,
+    RuntimeStats,
+}
+
 public record HeartbeatData(HeartbeatType Type);
 
 public record TaskHeartbeatEntry(
@@ -41,6 +54,16 @@ public record TaskHeartbeatEntry(
     Guid MachineId,
     HeartbeatData[] Data);
 
+public record JobResultData(JobResultType Type);
+
+public record TaskJobResultEntry(
+    Guid TaskId,
+    Guid? JobId,
+    Guid MachineId,
+    JobResultData Data,
+    Dictionary<string, double> Value
+    );
+
 public record NodeHeartbeatEntry(Guid NodeId, HeartbeatData[] Data);
 
 public record NodeCommandStopIfFree();
@@ -892,6 +915,27 @@ public record SecretAddress<T>(Uri Url) : ISecret<T> {
 public record SecretData<T>(ISecret<T> Secret) {
 }
 
+public record JobResult(
+    [PartitionKey][RowKey] Guid JobId,
+    string Project,
+    string Name,
+    double NewCrashingInput = 0,
+    double NoReproCrashingInput = 0,
+    double NewReport = 0,
+    double NewUniqueReport = 0,
+    double NewRegressionReport = 0,
+    double NewCrashDump = 0,
+    double InstructionsCovered = 0,
+    double TotalInstructions = 0,
+    double CoverageRate = 0,
+    double IterationCount = 0
+) : EntityBase() {
+    public JobResult(Guid JobId, string Project, string Name) : this(
+        JobId: JobId,
+        Project: Project,
+        Name: Name, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) { }
+}
+
 public record JobConfig(
     string Project,
     string Name,
@@ -1056,6 +1100,7 @@ public record TaskUnitConfig(
     string? InstanceTelemetryKey,
     string? MicrosoftTelemetryKey,
     Uri HeartbeatQueue,
+    Uri JobResultQueue,
     Dictionary<string, string> Tags
     ) {
     public Uri? inputQueue { get; set; }

src/ApiService/ApiService/Program.cs

@@ -118,6 +118,7 @@ public static async Async.Task Main() {
                 .AddScoped<IVmOperations, VmOperations>()
                 .AddScoped<ISecretsOperations, SecretsOperations>()
                 .AddScoped<IJobOperations, JobOperations>()
+                .AddScoped<IJobResultOperations, JobResultOperations>()
                 .AddScoped<INsgOperations, NsgOperations>()
                 .AddScoped<IScheduler, Scheduler>()
                 .AddScoped<IConfig, Config>()

src/ApiService/ApiService/onefuzzlib/AutoScale.cs

@@ -1,5 +1,4 @@
-using System.Net;
-using System.Text.Json;
+using System.Text.Json;
 using System.Threading.Tasks;
 using ApiService.OneFuzzLib.Orm;
 using Azure;
@@ -10,9 +9,7 @@
 
 namespace Microsoft.OneFuzz.Service;
 
-public interface IAutoScaleOperations {
-
-    public Async.Task<ResultVoid<(HttpStatusCode Status, string Reason)>> Insert(AutoScale autoScale);
+public interface IAutoScaleOperations : IOrm<AutoScale> {
 
     public Async.Task<AutoScale?> GetSettingsForScaleset(ScalesetId scalesetId);
 
@@ -263,8 +260,8 @@ private async Async.Task<OneFuzzResult<DiagnosticSettingsResource>> SetupAutoSca
             // The field is there in github though, so need to update this code once that code is released:
             // https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/monitor/Azure.ResourceManager.Monitor/src/Generated/Models/LogSettings.cs
             // But setting logs one by one works the same as "allLogs" being set...
-            var logSettings1 = new LogSettings(true) { RetentionPolicy = new RetentionPolicy(true, 30), Category = "AutoscaleEvaluations" };
-            var logSettings2 = new LogSettings(true) { RetentionPolicy = new RetentionPolicy(true, 30), Category = "AutoscaleScaleActions" };
+            var logSettings1 = new LogSettings(true) { Category = "AutoscaleEvaluations" };
+            var logSettings2 = new LogSettings(true) { Category = "AutoscaleScaleActions" };
 
             var parameters = new DiagnosticSettingsData {
                 WorkspaceId = logAnalyticsWorkspaceId

src/ApiService/ApiService/onefuzzlib/Config.cs

@@ -71,6 +71,7 @@ private static BlobContainerSasPermissions ConvertPermissions(ContainerPermissio
             InstanceTelemetryKey: _serviceConfig.ApplicationInsightsInstrumentationKey,
             MicrosoftTelemetryKey: _serviceConfig.OneFuzzTelemetry,
             HeartbeatQueue: await _queue.GetQueueSas("task-heartbeat", StorageType.Config, QueueSasPermissions.Add) ?? throw new Exception("unable to get heartbeat queue sas"),
+            JobResultQueue: await _queue.GetQueueSas("job-result", StorageType.Config, QueueSasPermissions.Add) ?? throw new Exception("unable to get heartbeat queue sas"),
             Tags: task.Config.Tags ?? new Dictionary<string, string>()
         );
 

src/ApiService/ApiService/onefuzzlib/Defs.cs

@@ -192,7 +192,7 @@ public static class Defs {
                         ),
                         new ContainerDefinition(
                             Type:ContainerType.Crashdumps,
-                            Compare: Compare.Equal,
+                            Compare: Compare.AtMost,
                             Value:1,
                             Permissions: ContainerPermission.Write
                             ),
@@ -287,7 +287,7 @@ public static class Defs {
                         ),
                         new ContainerDefinition(
                             Type:ContainerType.Crashdumps,
-                            Compare: Compare.Equal,
+                            Compare: Compare.AtMost,
                             Value:1,
                             Permissions: ContainerPermission.Write
                             ),