diff --git a/src/ApiService/ApiService/ApiService.csproj b/src/ApiService/ApiService/ApiService.csproj index ec2bc65a71..c1186f3b6c 100644 --- a/src/ApiService/ApiService/ApiService.csproj +++ b/src/ApiService/ApiService/ApiService.csproj @@ -25,6 +25,7 @@ + diff --git a/src/ApiService/ApiService/EnvironmentVariables.cs b/src/ApiService/ApiService/EnvironmentVariables.cs index 1fee50ceff..9ad665e8ce 100644 --- a/src/ApiService/ApiService/EnvironmentVariables.cs +++ b/src/ApiService/ApiService/EnvironmentVariables.cs @@ -1,5 +1,4 @@ -using System; -namespace Microsoft.OneFuzz.Service; +namespace Microsoft.OneFuzz.Service; public enum LogDestination { diff --git a/src/ApiService/ApiService/HttpClient.cs b/src/ApiService/ApiService/HttpClient.cs index b931215dd7..d268e609d8 100644 --- a/src/ApiService/ApiService/HttpClient.cs +++ b/src/ApiService/ApiService/HttpClient.cs @@ -1,6 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; +using System.Net.Http; using System.Threading.Tasks; using System.Net.Http.Headers; diff --git a/src/ApiService/ApiService/Log.cs b/src/ApiService/ApiService/Log.cs index e913ff3538..006a364df9 100644 --- a/src/ApiService/ApiService/Log.cs +++ b/src/ApiService/ApiService/Log.cs @@ -1,6 +1,4 @@ -using System; -using System.Collections.Generic; -using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.ApplicationInsights.DataContracts; diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index ea5be2c749..e45946d2da 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -111,7 +111,7 @@ public enum TaskDebugFlag KeepNodeOnCompletion, } -public enum ScalesetState +public enum ScalesetState { Init, Setup, @@ -122,13 +122,13 @@ public enum ScalesetState CreationFailed } -public static class ScalesetStateHelper +public static class ScalesetStateHelper { static ConcurrentDictionary _states = new ConcurrentDictionary(); /// set of states that indicate the scaleset can be updated - public static ScalesetState[] CanUpdate() + public static ScalesetState[] CanUpdate() { return _states.GetOrAdd("CanUpdate", k => new[]{ @@ -138,7 +138,7 @@ public static ScalesetState[] CanUpdate() } /// set of states that indicate work is needed during eventing - public static ScalesetState[] NeedsWork() + public static ScalesetState[] NeedsWork() { return _states.GetOrAdd("CanUpdate", k => new[]{ @@ -151,10 +151,10 @@ public static ScalesetState[] NeedsWork() } /// set of states that indicate if it's available for work - public static ScalesetState[] Available() + public static ScalesetState[] Available() { return - _states.GetOrAdd("CanUpdate", k => + _states.GetOrAdd("CanUpdate", k => { return new[]{ @@ -165,10 +165,10 @@ public static ScalesetState[] Available() } /// set of states that indicate scaleset is resizing - public static ScalesetState[] Resizing() + public static ScalesetState[] Resizing() { return - _states.GetOrAdd("CanDelete", k => + _states.GetOrAdd("CanDelete", k => { return new[]{ @@ -178,6 +178,24 @@ public static ScalesetState[] Resizing() }; }); } +} - -} \ No newline at end of file +public static class TaskStateHelper +{ + static ConcurrentDictionary _states = new ConcurrentDictionary(); + public static TaskState[] Available() + { + return + _states.GetOrAdd("Available", k => + { + return + new[]{ + TaskState.Waiting, + TaskState.Scheduled, + TaskState.SettingUp, + TaskState.Running, + TaskState.WaitJob + }; + }); + } +} diff --git a/src/ApiService/ApiService/OneFuzzTypes/Events.cs b/src/ApiService/ApiService/OneFuzzTypes/Events.cs index a83afaf5a1..6ec095b4c9 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Events.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Events.cs @@ -1,5 +1,4 @@ -using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using System; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; using System.Text.Json; using System.Text.Json.Serialization; using PoolName = System.String; @@ -37,7 +36,7 @@ public enum EventType FileAdded, TaskHeartbeat, NodeHeartbeat, - InstanceConfigUpdated + InstanceConfigUpdated, } public abstract record BaseEvent() @@ -50,6 +49,9 @@ public EventType GetEventType() EventNodeHeartbeat _ => EventType.NodeHeartbeat, EventTaskHeartbeat _ => EventType.TaskHeartbeat, EventInstanceConfigUpdated _ => EventType.InstanceConfigUpdated, + EventCrashReported _ => EventType.CrashReported, + EventRegressionReported _ => EventType.RegressionReported, + EventFileAdded _ => EventType.FileAdded, _ => throw new NotImplementedException(), }; @@ -62,6 +64,9 @@ public static Type GetTypeInfo(EventType eventType) EventType.NodeHeartbeat => typeof(EventNodeHeartbeat), EventType.InstanceConfigUpdated => typeof(EventInstanceConfigUpdated), EventType.TaskHeartbeat => typeof(EventTaskHeartbeat), + EventType.CrashReported => typeof(EventCrashReported), + EventType.RegressionReported => typeof(EventRegressionReported), + EventType.FileAdded => typeof(EventFileAdded), _ => throw new ArgumentException($"invalid input {eventType}"), }; @@ -249,25 +254,25 @@ PoolName PoolName // NodeState state // ) : BaseEvent(); -// record EventCrashReported( -// Report Report, -// Container Container, -// [property: JsonPropertyName("filename")] String FileName, -// TaskConfig? TaskConfig -// ) : BaseEvent(); +record EventCrashReported( + Report Report, + Container Container, + [property: JsonPropertyName("filename")] String FileName, + TaskConfig? TaskConfig +) : BaseEvent(); -// record EventRegressionReported( -// RegressionReport RegressionReport, -// Container Container, -// [property: JsonPropertyName("filename")] String FileName, -// TaskConfig? TaskConfig -// ) : BaseEvent(); +record EventRegressionReported( + RegressionReport RegressionReport, + Container Container, + [property: JsonPropertyName("filename")] String FileName, + TaskConfig? TaskConfig +) : BaseEvent(); -// record EventFileAdded( -// Container Container, -// [property: JsonPropertyName("filename")] String FileName -// ) : BaseEvent(); +record EventFileAdded( + Container Container, + [property: JsonPropertyName("filename")] String FileName +) : BaseEvent(); public record EventInstanceConfigUpdated( @@ -296,4 +301,4 @@ public override void Write(Utf8JsonWriter writer, BaseEvent value, JsonSerialize var eventType = value.GetType(); JsonSerializer.Serialize(writer, value, eventType, options); } -} \ No newline at end of file +} diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index b64647f41d..bd0d6a5179 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -1,9 +1,5 @@ using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using System; -using System.Collections.Generic; using System.Text.Json.Serialization; - -using Container = System.String; using Region = System.String; using PoolName = System.String; using Endpoint = System.String; @@ -397,32 +393,74 @@ Dictionary Tags ) : EntityBase(); +public record Container(string ContainerName) +{ + public string ContainerName { get; } = ContainerName.All(c => char.IsLetterOrDigit(c) || c == '-') ? ContainerName : throw new ArgumentException("Container name must have only numbers, letters or dashes"); +} + +public record Notification( + DateTime? Timestamp, + Container Container, + Guid NotificationId, + NotificationTemplate Config +) : EntityBase(); public record BlobRef( string Account, - Container Container, - string Name + Container container, + string name ); - public record Report( - string? InputURL, + string? InputUrl, BlobRef? InputBlob, - string? Executable, + string Executable, string CrashType, string CrashSite, List CallStack, string CallStackSha256, string InputSha256, string? AsanLog, - Guid TaskID, - Guid JobID, + Guid TaskId, + Guid JobId, int? ScarinessScore, string? ScarinessDescription, - List MinimizedStack, + List? MinimizedStack, string? MinimizedStackSha256, - List MinimizedStackFunctionNames, + List? MinimizedStackFunctionNames, string? MinimizedStackFunctionNamesSha256, - List MinimizedStackFunctionLines, + List? MinimizedStackFunctionLines, string? MinimizedStackFunctionLinesSha256 ); + +public record NoReproReport( + string InputSha, + BlobRef? InputBlob, + string? Executable, + Guid TaskId, + Guid JobId, + int Tries, + string? Error +); + +public record CrashTestResult( + Report? CrashReport, + NoReproReport? NoReproReport +); + +public record RegressionReport( + CrashTestResult CrashTestResult, + CrashTestResult? OriginalCrashTestResult +); + +public record NotificationTemplate( + AdoTemplate? AdoTemplate, + TeamsTemplate? TeamsTemplate, + GithubIssuesTemplate? GithubIssuesTemplate +); + +public record AdoTemplate(); + +public record TeamsTemplate(); + +public record GithubIssuesTemplate(); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs b/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs index 16d453728c..b6f054b5a1 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs @@ -1,6 +1,4 @@ using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using System; -using System.Collections.Generic; using System.Text.Json.Serialization; namespace Microsoft.OneFuzz.Service; diff --git a/src/ApiService/ApiService/Program.cs b/src/ApiService/ApiService/Program.cs index 4532f60c82..7f912b7e68 100644 --- a/src/ApiService/ApiService/Program.cs +++ b/src/ApiService/ApiService/Program.cs @@ -1,8 +1,10 @@ // to avoid collision with Task in model.cs global using Async = System.Threading.Tasks; -using System; -using System.Collections.Generic; +global using System; +global using System.Collections.Generic; +global using System.Linq; + using Microsoft.Extensions.Hosting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Azure.Functions.Worker.Middleware; @@ -76,6 +78,9 @@ public static void Main() .AddScoped() .AddScoped() .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() //TODO: move out expensive resources into separate class, and add those as Singleton // ArmClient, Table Client(s), Queue Client(s), HttpClient, etc. diff --git a/src/ApiService/ApiService/QueueFileChanges.cs b/src/ApiService/ApiService/QueueFileChanges.cs index c106d986ef..d70b9096fe 100644 --- a/src/ApiService/ApiService/QueueFileChanges.cs +++ b/src/ApiService/ApiService/QueueFileChanges.cs @@ -1,9 +1,6 @@ -using System; using Microsoft.Azure.Functions.Worker; -using System.Collections.Generic; using System.Text.Json; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using System.Linq; namespace Microsoft.OneFuzz.Service; @@ -17,51 +14,53 @@ public class QueueFileChanges private readonly IStorage _storage; - public QueueFileChanges(ILogTracer log, IStorage storage) + private readonly INotificationOperations _notificationOperations; + + public QueueFileChanges(ILogTracer log, IStorage storage, INotificationOperations notificationOperations) { _log = log; _storage = storage; + _notificationOperations = notificationOperations; } [Function("QueueFileChanges")] - public Async.Task Run( + public async Async.Task Run( [QueueTrigger("file-changes-refactored", Connection = "AzureWebJobsStorage")] string msg, int dequeueCount) { - var fileChangeEvent = JsonSerializer.Deserialize>(msg, EntityConverter.GetJsonSerializerOptions()); + var fileChangeEvent = JsonSerializer.Deserialize(msg, EntityConverter.GetJsonSerializerOptions()); var lastTry = dequeueCount == MAX_DEQUEUE_COUNT; var _ = fileChangeEvent ?? throw new ArgumentException("Unable to parse queue trigger as JSON"); // check type first before calling Azure APIs const string eventType = "eventType"; - if (!fileChangeEvent.ContainsKey(eventType) - || fileChangeEvent[eventType] != "Microsoft.Storage.BlobCreated") + if (!fileChangeEvent.RootElement.TryGetProperty(eventType, out var eventTypeElement) + || eventTypeElement.GetString() != "Microsoft.Storage.BlobCreated") { - return Async.Task.CompletedTask; + return; } const string topic = "topic"; - if (!fileChangeEvent.ContainsKey(topic) - || !_storage.CorpusAccounts().Contains(fileChangeEvent[topic])) + if (!fileChangeEvent.RootElement.TryGetProperty(topic, out var topicElement) + || !_storage.CorpusAccounts().Contains(topicElement.GetString())) { - return Async.Task.CompletedTask; + return; } - file_added(_log, fileChangeEvent, lastTry); - return Async.Task.CompletedTask; + await file_added(_log, fileChangeEvent, lastTry); } - private void file_added(ILogTracer log, Dictionary fileChangeEvent, bool failTaskOnTransientError) + private async Async.Task file_added(ILogTracer log, JsonDocument fileChangeEvent, bool failTaskOnTransientError) { - var data = JsonSerializer.Deserialize>(fileChangeEvent["data"])!; - var url = data["url"]; + var data = fileChangeEvent.RootElement.GetProperty("data"); + var url = data.GetProperty("url").GetString()!; var parts = url.Split("/").Skip(3).ToList(); var container = parts[0]; var path = string.Join('/', parts.Skip(1)); log.Info($"file added container: {container} - path: {path}"); - // TODO: new_files(container, path, fail_task_on_transient_error) + await _notificationOperations.NewFiles(new Container(container), path, failTaskOnTransientError); } } diff --git a/src/ApiService/ApiService/QueueNodeHearbeat.cs b/src/ApiService/ApiService/QueueNodeHearbeat.cs index b0425d8c15..247bd1aa4b 100644 --- a/src/ApiService/ApiService/QueueNodeHearbeat.cs +++ b/src/ApiService/ApiService/QueueNodeHearbeat.cs @@ -1,4 +1,3 @@ -using System; using Microsoft.Azure.Functions.Worker; using System.Text.Json; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; diff --git a/src/ApiService/ApiService/QueueProxyHeartbeat.cs b/src/ApiService/ApiService/QueueProxyHeartbeat.cs index 198badddc0..6b963f5847 100644 --- a/src/ApiService/ApiService/QueueProxyHeartbeat.cs +++ b/src/ApiService/ApiService/QueueProxyHeartbeat.cs @@ -1,4 +1,3 @@ -using System; using Microsoft.Azure.Functions.Worker; using System.Text.Json; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; diff --git a/src/ApiService/ApiService/QueueTaskHearbeat.cs b/src/ApiService/ApiService/QueueTaskHearbeat.cs index a8d02ffffa..f6af56e05b 100644 --- a/src/ApiService/ApiService/QueueTaskHearbeat.cs +++ b/src/ApiService/ApiService/QueueTaskHearbeat.cs @@ -1,4 +1,3 @@ -using System; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; using System.Text.Json; diff --git a/src/ApiService/ApiService/TestHooks.cs b/src/ApiService/ApiService/TestHooks.cs index 73efc2b618..37b85032fd 100644 --- a/src/ApiService/ApiService/TestHooks.cs +++ b/src/ApiService/ApiService/TestHooks.cs @@ -1,5 +1,4 @@ -using System; -using System.Net; +using System.Net; using System.Threading.Tasks; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; diff --git a/src/ApiService/ApiService/UserCredentials.cs b/src/ApiService/ApiService/UserCredentials.cs index d4b647d3dc..22b0ce3027 100644 --- a/src/ApiService/ApiService/UserCredentials.cs +++ b/src/ApiService/ApiService/UserCredentials.cs @@ -1,6 +1,4 @@ -using System; -using System.Linq; -using System.Net.Http.Headers; +using System.Net.Http.Headers; using System.Threading.Tasks; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.IdentityModel.Tokens; diff --git a/src/ApiService/ApiService/onefuzzlib/Containers.cs b/src/ApiService/ApiService/onefuzzlib/Containers.cs new file mode 100644 index 0000000000..de416ebdc2 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/Containers.cs @@ -0,0 +1,93 @@ +using System.Threading.Tasks; +using Azure.ResourceManager; +using Azure.Storage.Blobs; +using Azure.Storage; +using Azure; + +namespace Microsoft.OneFuzz.Service; + +public interface IContainers +{ + public Task?> GetBlob(Container container, string name, StorageType storageType); + + public Async.Task FindContainer(Container container, StorageType storageType); + + public Uri GetFileSasUrl(Container container, string name, StorageType storageType, bool read = false, bool add = false, bool create = false, bool write = false, bool delete = false, bool delete_previous_version = false, bool tag = false, int days = 30, int hours = 0, int minutes = 0); + +} + +public class Containers : IContainers +{ + private ILogTracer _log; + private IStorage _storage; + private ICreds _creds; + private ArmClient _armClient; + public Containers(ILogTracer log, IStorage storage, ICreds creds) + { + _log = log; + _storage = storage; + _creds = creds; + _armClient = new ArmClient(credential: _creds.GetIdentity(), defaultSubscriptionId: _creds.GetSubcription()); + } + public async Task?> GetBlob(Container container, string name, StorageType storageType) + { + var client = await FindContainer(container, storageType); + + if (client == null) + { + return null; + } + + try + { + return (await client.GetBlobClient(name).DownloadContentAsync()) + .Value.Content.ToArray(); + } + catch (RequestFailedException) + { + return null; + } + } + + public async Async.Task FindContainer(Container container, StorageType storageType) + { + // # check secondary accounts first by searching in reverse. + // # + // # By implementation, the primary account is specified first, followed by + // # any secondary accounts. + // # + // # Secondary accounts, if they exist, are preferred for containers and have + // # increased IOP rates, this should be a slight optimization + return await _storage.GetAccounts(storageType) + .Reverse() + .Select(account => GetBlobService(account)?.GetBlobContainerClient(container.ContainerName)) + .ToAsyncEnumerable() + .WhereAwait(async client => client != null && (await client.ExistsAsync()).Value) + .FirstOrDefaultAsync(); + } + + private BlobServiceClient? GetBlobService(string accountId) + { + _log.Info($"getting blob container (account_id: {accountId}"); + var (accountName, accountKey) = _storage.GetStorageAccountNameAndKey(accountId); + if (accountName == null) + { + _log.Error("Failed to get storage account name"); + return null; + } + var storageKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); + var accountUrl = GetUrl(accountName); + return new BlobServiceClient(accountUrl, storageKeyCredential); + } + + private static Uri GetUrl(string accountName) + { + return new Uri($"https://{accountName}.blob.core.windows.net/"); + } + + public Uri GetFileSasUrl(Container container, string name, StorageType storageType, bool read = false, bool add = false, bool create = false, bool write = false, bool delete = false, bool delete_previous_version = false, bool tag = false, int days = 30, int hours = 0, int minutes = 0) + { + throw new NotImplementedException(); + } +} + diff --git a/src/ApiService/ApiService/onefuzzlib/Events.cs b/src/ApiService/ApiService/onefuzzlib/Events.cs index c31f9414da..73aa1971db 100644 --- a/src/ApiService/ApiService/onefuzzlib/Events.cs +++ b/src/ApiService/ApiService/onefuzzlib/Events.cs @@ -1,6 +1,4 @@ using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using System; -using System.Collections.Generic; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; diff --git a/src/ApiService/ApiService/onefuzzlib/InstanceConfig.cs b/src/ApiService/ApiService/onefuzzlib/InstanceConfig.cs index 66ba7e72db..74480a7a1e 100644 --- a/src/ApiService/ApiService/onefuzzlib/InstanceConfig.cs +++ b/src/ApiService/ApiService/onefuzzlib/InstanceConfig.cs @@ -1,5 +1,4 @@ using ApiService.OneFuzzLib.Orm; -using System; using System.Threading.Tasks; namespace Microsoft.OneFuzz.Service; diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs index 5a42576aea..be6f6a6612 100644 --- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs @@ -1,6 +1,4 @@ using ApiService.OneFuzzLib.Orm; -using System; -using System.Linq; using System.Threading.Tasks; namespace Microsoft.OneFuzz.Service; diff --git a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs new file mode 100644 index 0000000000..66bf8c6e6a --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs @@ -0,0 +1,146 @@ +using System.Text.Json; +using ApiService.OneFuzzLib.Orm; + +namespace Microsoft.OneFuzz.Service; + +public interface INotificationOperations +{ + Async.Task NewFiles(Container container, string filename, bool failTaskOnTransientError); +} + +public class NotificationOperations : Orm, INotificationOperations +{ + private ILogTracer _log; + private IReports _reports; + private ITaskOperations _taskOperations; + + private IContainers _containers; + + private IQueue _queue; + + private IEvents _events; + + public NotificationOperations(ILogTracer log, IStorage storage, IReports reports, ITaskOperations taskOperations, IContainers containers, IQueue queue, IEvents events) + : base(storage) + { + _log = log; + _reports = reports; + _taskOperations = taskOperations; + _containers = containers; + _queue = queue; + _events = events; + } + public async Async.Task NewFiles(Container container, string filename, bool failTaskOnTransientError) + { + var notifications = GetNotifications(container); + var hasNotifications = await notifications.AnyAsync(); + var report = await _reports.GetReportOrRegression(container, filename, expectReports: hasNotifications); + + if (!hasNotifications) + { + return; + } + + var done = new List(); + await foreach (var notification in notifications) + { + if (done.Contains(notification.Config)) + { + continue; + } + + done.Add(notification.Config); + + if (notification.Config.TeamsTemplate != null) + { + NotifyTeams(notification.Config.TeamsTemplate, container, filename, report); + } + + if (report == null) + { + continue; + } + + if (notification.Config.AdoTemplate != null) + { + NotifyAdo(notification.Config.AdoTemplate, container, filename, report, failTaskOnTransientError); + } + + if (notification.Config.GithubIssuesTemplate != null) + { + GithubIssue(notification.Config.GithubIssuesTemplate, container, filename, report); + } + } + + await foreach (var (task, containers) in GetQueueTasks()) + { + if (containers.Contains(container.ContainerName)) + { + _log.Info($"queuing input {container.ContainerName} {filename} {task.TaskId}"); + var url = _containers.GetFileSasUrl(container, filename, StorageType.Corpus, read: true, delete: true); + await _queue.SendMessage(task.TaskId.ToString(), System.Text.Encoding.UTF8.GetBytes(url.ToString()), StorageType.Corpus); + } + } + + if (report == null) + { + await _events.SendEvent(new EventFileAdded(container, filename)); + } + else if (report.Report != null) + { + var reportTask = await _taskOperations.GetByJobIdAndTaskId(report.Report.JobId, report.Report.TaskId); + + var crashReportedEvent = new EventCrashReported(report.Report, container, filename, reportTask?.Config); + await _events.SendEvent(crashReportedEvent); + } + else if (report.RegressionReport != null) + { + var reportTask = await GetRegressionReportTask(report.RegressionReport); + + var regressionEvent = new EventRegressionReported(report.RegressionReport, container, filename, reportTask?.Config); + } + } + + public IAsyncEnumerable GetNotifications(Container container) + { + return QueryAsync(filter: $"container eq '{container.ContainerName}'"); + } + + public IAsyncEnumerable<(Task, IEnumerable)> GetQueueTasks() + { + // Nullability mismatch: We filter tuples where the containers are null + return _taskOperations.SearchStates(states: TaskStateHelper.Available()) + .Select(task => (task, _taskOperations.GetInputContainerQueues(task.Config))) + .Where(taskTuple => taskTuple.Item2 != null)!; + } + + private async Async.Task GetRegressionReportTask(RegressionReport report) + { + if (report.CrashTestResult.CrashReport != null) + { + return await _taskOperations.GetByJobIdAndTaskId(report.CrashTestResult.CrashReport.JobId, report.CrashTestResult.CrashReport.TaskId); + } + if (report.CrashTestResult.NoReproReport != null) + { + return await _taskOperations.GetByJobIdAndTaskId(report.CrashTestResult.NoReproReport.JobId, report.CrashTestResult.NoReproReport.TaskId); + } + + _log.Error($"unable to find crash_report or no repro entry for report: {JsonSerializer.Serialize(report)}"); + return null; + } + + private void GithubIssue(GithubIssuesTemplate config, Container container, string filename, RegressionReportOrReport? report) + { + throw new NotImplementedException(); + } + + private void NotifyAdo(AdoTemplate config, Container container, string filename, RegressionReportOrReport report, bool failTaskOnTransientError) + { + throw new NotImplementedException(); + } + + private void NotifyTeams(TeamsTemplate config, Container container, string filename, RegressionReportOrReport? report) + { + throw new NotImplementedException(); + } +} diff --git a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs index e6bcbe0c64..a087a67e77 100644 --- a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs @@ -1,6 +1,4 @@ using ApiService.OneFuzzLib.Orm; -using System; -using System.Linq; using System.Threading.Tasks; namespace Microsoft.OneFuzz.Service; diff --git a/src/ApiService/ApiService/onefuzzlib/Queue.cs b/src/ApiService/ApiService/onefuzzlib/Queue.cs index 5b0f823241..bce5fb786d 100644 --- a/src/ApiService/ApiService/onefuzzlib/Queue.cs +++ b/src/ApiService/ApiService/onefuzzlib/Queue.cs @@ -1,7 +1,6 @@ using Azure.Storage; using Azure.Storage.Queues; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using System; using System.Text.Json; using System.Threading.Tasks; diff --git a/src/ApiService/ApiService/onefuzzlib/Reports.cs b/src/ApiService/ApiService/onefuzzlib/Reports.cs new file mode 100644 index 0000000000..a60ad3d16b --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/Reports.cs @@ -0,0 +1,98 @@ +using System.Text.Json; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +namespace Microsoft.OneFuzz.Service; + +public interface IReports +{ + public Async.Task GetReportOrRegression(Container container, string fileName, bool expectReports = false, params string[] args); +} + +public class Reports : IReports +{ + private ILogTracer _log; + private IContainers _containers; + public Reports(ILogTracer log, IContainers containers) + { + _log = log; + _containers = containers; + } + + public async Async.Task GetReportOrRegression(Container container, string fileName, bool expectReports = false, params string[] args) + { + var filePath = String.Join("/", new[] { container.ContainerName, fileName }); + if (!fileName.EndsWith(".json")) + { + if (expectReports) + { + _log.Error($"get_report invalid extension: {filePath}"); + } + return null; + } + + var blob = await _containers.GetBlob(container, fileName, StorageType.Corpus); + + if (blob == null) + { + if (expectReports) + { + _log.Error($"get_report invalid blob: {filePath}"); + } + return null; + } + + return ParseReportOrRegression(blob, filePath, expectReports); + } + + private RegressionReportOrReport? ParseReportOrRegression(string content, string? filePath, bool expectReports = false) + { + try + { + return new RegressionReportOrReport + { + RegressionReport = JsonSerializer.Deserialize(content, EntityConverter.GetJsonSerializerOptions()) + }; + } + catch (JsonException e) + { + try + { + return new RegressionReportOrReport + { + Report = JsonSerializer.Deserialize(content, EntityConverter.GetJsonSerializerOptions()) + }; + } + catch (JsonException e2) + { + if (expectReports) + { + _log.Error($"unable to parse report ({filePath}) as a report or regression. regression error: {e.Message} report error: {e2.Message}"); + } + return null; + } + } + } + + private RegressionReportOrReport? ParseReportOrRegression(IEnumerable content, string? filePath, bool expectReports = false) + { + try + { + var str = System.Text.Encoding.UTF8.GetString(content.ToArray()); + return ParseReportOrRegression(str, filePath, expectReports); + } + catch (Exception e) + { + if (expectReports) + { + _log.Error($"unable to parse report ({filePath}): unicode decode of report failed - {e.Message} {e.StackTrace}"); + } + return null; + } + } +} + +public class RegressionReportOrReport +{ + public RegressionReport? RegressionReport { get; set; } + public Report? Report { get; set; } +} diff --git a/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs b/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs index 858987b512..94ecd94926 100644 --- a/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs @@ -1,5 +1,4 @@ using ApiService.OneFuzzLib.Orm; -using System.Collections.Generic; namespace Microsoft.OneFuzz.Service; diff --git a/src/ApiService/ApiService/onefuzzlib/Storage.cs b/src/ApiService/ApiService/onefuzzlib/Storage.cs index ffafb282bd..730ed9a9c9 100644 --- a/src/ApiService/ApiService/onefuzzlib/Storage.cs +++ b/src/ApiService/ApiService/onefuzzlib/Storage.cs @@ -1,10 +1,7 @@ -using System.Collections.Generic; -using System; using Azure.ResourceManager; using Azure.ResourceManager.Storage; using Azure.Core; using System.Text.Json; -using System.Linq; namespace Microsoft.OneFuzz.Service; @@ -21,6 +18,8 @@ public interface IStorage public IEnumerable CorpusAccounts(); string GetPrimaryAccount(StorageType storageType); public (string?, string?) GetStorageAccountNameAndKey(string accountId); + + public IEnumerable GetAccounts(StorageType storageType); } public class Storage : IStorage @@ -114,4 +113,17 @@ public string GetPrimaryAccount(StorageType storageType) var key = storageAccount.GetKeys().Value.Keys.FirstOrDefault(); return (resourceId.Name, key?.Value); } + + public IEnumerable GetAccounts(StorageType storageType) + { + switch (storageType) + { + case StorageType.Corpus: + return CorpusAccounts(); + case StorageType.Config: + return new[] { GetFuncStorage() }; + default: + throw new NotImplementedException(); + } + } } diff --git a/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs b/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs index ee056cd4d9..18c30aa0de 100644 --- a/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs @@ -1,12 +1,18 @@ using ApiService.OneFuzzLib.Orm; -using System; -using System.Linq; namespace Microsoft.OneFuzz.Service; public interface ITaskOperations : IOrm { Async.Task GetByTaskId(Guid taskId); + + Async.Task GetByJobIdAndTaskId(Guid jobId, Guid taskId); + + + IAsyncEnumerable SearchStates(Guid? jobId = null, IEnumerable? states = null); + + IEnumerable? GetInputContainerQueues(TaskConfig config); + } public class TaskOperations : Orm, ITaskOperations @@ -25,4 +31,37 @@ public TaskOperations(IStorage storage) return await data.FirstOrDefaultAsync(); } + public async Async.Task GetByJobIdAndTaskId(Guid jobId, Guid taskId) + { + var data = QueryAsync(filter: $"PartitionKey eq '{jobId}' and RowKey eq '{taskId}'"); + + return await data.FirstOrDefaultAsync(); + } + public IAsyncEnumerable SearchStates(Guid? jobId = null, IEnumerable? states = null) + { + var queryString = String.Empty; + if (jobId != null) + { + queryString += $"PartitionKey eq '{jobId}'"; + } + + if (states != null) + { + if (jobId != null) + { + queryString += " and "; + } + + var statesString = string.Join(",", states); + queryString += $"state in ({statesString})"; + } + + return QueryAsync(filter: queryString); + } + + public IEnumerable? GetInputContainerQueues(TaskConfig config) + { + throw new NotImplementedException(); + } + } diff --git a/src/ApiService/ApiService/onefuzzlib/Utils.cs b/src/ApiService/ApiService/onefuzzlib/Utils.cs index e5eb7d8786..0be4062275 100644 --- a/src/ApiService/ApiService/onefuzzlib/Utils.cs +++ b/src/ApiService/ApiService/onefuzzlib/Utils.cs @@ -1,6 +1,3 @@ - -using System; - namespace Microsoft.OneFuzz.Service; public static class ObjectExtention diff --git a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs index f28e02448b..1ebe6de410 100644 --- a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs @@ -1,6 +1,4 @@ using ApiService.OneFuzzLib.Orm; -using System; -using System.Collections.Generic; namespace Microsoft.OneFuzz.Service; diff --git a/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs b/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs index e745c77ffb..2cd734e254 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; public class CaseConverter { diff --git a/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs b/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs index a77233707e..97fb4f30a5 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; +using System.Diagnostics; using System.Reflection; using System.Runtime.CompilerServices; using System.Text.Encodings.Web; diff --git a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs index f92b134c59..fabfff10ec 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs @@ -1,13 +1,10 @@ using Azure.Data.Tables; -using System; using System.Reflection; -using System.Linq; using System.Linq.Expressions; using System.Text.Json; using System.Text.Json.Serialization; using System.Collections.Concurrent; using Azure; -using System.Collections.Generic; namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; diff --git a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs index 26d9acfbfd..531c779a62 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs @@ -1,9 +1,6 @@ using Azure.Data.Tables; using Microsoft.OneFuzz.Service; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using System; -using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; namespace ApiService.OneFuzzLib.Orm diff --git a/src/ApiService/ApiService/packages.lock.json b/src/ApiService/ApiService/packages.lock.json index 885036b6ad..80de00828c 100644 --- a/src/ApiService/ApiService/packages.lock.json +++ b/src/ApiService/ApiService/packages.lock.json @@ -107,6 +107,16 @@ "System.Text.Json": "4.7.2" } }, + "Azure.Storage.Blobs": { + "type": "Direct", + "requested": "[12.11.0, )", + "resolved": "12.11.0", + "contentHash": "50eRjIhY7Q1JN7kT2MSawDKCcwSb7uRZUkz00P/BLjSg47gm2hxUYsnJPyvzCHntYMbOWzrvaVQTwYwXabaR5Q==", + "dependencies": { + "Azure.Storage.Common": "12.10.0", + "System.Text.Json": "4.7.2" + } + }, "Azure.Storage.Queues": { "type": "Direct", "requested": "[12.9.0, )", diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index 5f423f43c2..8e1d41a735 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -249,6 +249,41 @@ public static Gen WebhookMessage() ); } + + public static Gen Report() + { + return Arb.Generate, Guid, int>>().Select( + arg => + new Report( + InputUrl: arg.Item1, + InputBlob: arg.Item2, + Executable: arg.Item1, + CrashType: arg.Item1, + CrashSite: arg.Item1, + CallStack: arg.Item3, + CallStackSha256: arg.Item1, + InputSha256: arg.Item1, + AsanLog: arg.Item1, + TaskId: arg.Item4, + JobId: arg.Item4, + ScarinessScore: arg.Item5, + ScarinessDescription: arg.Item1, + MinimizedStack: arg.Item3, + MinimizedStackSha256: arg.Item1, + MinimizedStackFunctionNames: arg.Item3, + MinimizedStackFunctionNamesSha256: arg.Item1, + MinimizedStackFunctionLines: arg.Item3, + MinimizedStackFunctionLinesSha256: arg.Item1 + ) + ); + } + + public static Gen Container() + { + return Arb.Generate>>().Select( + arg => new Container(string.Join("", arg.Item1.Get.Where(c => char.IsLetterOrDigit(c) || c == '-'))!) + ); + } } public class OrmArb @@ -327,6 +362,16 @@ public static Arbitrary WebhookMessage() { return Arb.From(OrmGenerators.WebhookMessage()); } + + public static Arbitrary Report() + { + return Arb.From(OrmGenerators.Report()); + } + + public static Arbitrary Container() + { + return Arb.From(OrmGenerators.Container()); + } }