Implement new_files (#1794)

* Checkpoint

* Checkpoint

* More merge resolving

* Code complete

* Tested that it works

* Keep the queue name different for now

* Query was wrong, should be and

* Style

* Fix compile issue

* Change report to use string instead of SHA, fixes tests as well

* PR comments

* Comments and formatting

src/ApiService/ApiService/ApiService.csproj

@@ -25,6 +25,7 @@
     <PackageReference Include="Azure.ResourceManager.Resources" Version="1.0.0" />
     <PackageReference Include="Azure.ResourceManager.Storage" Version="1.0.0-beta.8" />
     <PackageReference Include="Azure.Storage.Queues" Version="12.9.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.11.0" />
     <PackageReference Include="Microsoft.Graph" Version="4.24.0" />
     <PackageReference Include="Microsoft.Identity.Client" Version="4.43.0" />
     <PackageReference Include="Microsoft.Identity.Web.TokenCache" Version="1.23.1" />

src/ApiService/ApiService/EnvironmentVariables.cs

@@ -1,5 +1,4 @@
-using System;
-namespace Microsoft.OneFuzz.Service;
+namespace Microsoft.OneFuzz.Service;
 
 public enum LogDestination
 {

src/ApiService/ApiService/HttpClient.cs

@@ -1,6 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Net.Http;
+using System.Net.Http;
 using System.Threading.Tasks;
 using System.Net.Http.Headers;
 

src/ApiService/ApiService/Log.cs

@@ -1,6 +1,4 @@
-using System;
-using System.Collections.Generic;
-using Microsoft.ApplicationInsights;
+using Microsoft.ApplicationInsights;
 using Microsoft.ApplicationInsights.Extensibility;
 using Microsoft.ApplicationInsights.DataContracts;
 

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -111,7 +111,7 @@ public enum TaskDebugFlag
     KeepNodeOnCompletion,
 }
 
-public enum ScalesetState
+public enum ScalesetState
 {
     Init,
     Setup,
@@ -122,13 +122,13 @@ public enum ScalesetState
     CreationFailed
 }
 
-public static class ScalesetStateHelper
+public static class ScalesetStateHelper
 {
 
     static ConcurrentDictionary<string, ScalesetState[]> _states = new ConcurrentDictionary<string, ScalesetState[]>();
 
     /// set of states that indicate the scaleset can be updated
-    public static ScalesetState[] CanUpdate()
+    public static ScalesetState[] CanUpdate()
     {
         return
         _states.GetOrAdd("CanUpdate", k => new[]{
@@ -138,7 +138,7 @@ public static ScalesetState[] CanUpdate()
     }
 
     /// set of states that indicate work is needed during eventing
-    public static ScalesetState[] NeedsWork()
+    public static ScalesetState[] NeedsWork()
     {
         return
         _states.GetOrAdd("CanUpdate", k => new[]{
@@ -151,10 +151,10 @@ public static ScalesetState[] NeedsWork()
     }
 
     /// set of states that indicate if it's available for work
-    public static ScalesetState[] Available()
+    public static ScalesetState[] Available()
     {
         return
-        _states.GetOrAdd("CanUpdate", k =>
+        _states.GetOrAdd("CanUpdate", k =>
         {
             return
                 new[]{
@@ -165,10 +165,10 @@ public static ScalesetState[] Available()
     }
 
     /// set of states that indicate scaleset is resizing
-    public static ScalesetState[] Resizing()
+    public static ScalesetState[] Resizing()
     {
         return
-        _states.GetOrAdd("CanDelete", k =>
+        _states.GetOrAdd("CanDelete", k =>
         {
             return
                 new[]{
@@ -178,6 +178,24 @@ public static ScalesetState[] Resizing()
             };
         });
     }
+}
 
-
-}
+public static class TaskStateHelper
+{
+    static ConcurrentDictionary<string, TaskState[]> _states = new ConcurrentDictionary<string, TaskState[]>();
+    public static TaskState[] Available()
+    {
+        return
+        _states.GetOrAdd("Available", k =>
+        {
+            return
+                 new[]{
+                    TaskState.Waiting,
+                    TaskState.Scheduled,
+                    TaskState.SettingUp,
+                    TaskState.Running,
+                    TaskState.WaitJob
+                 };
+        });
+    }
+}

src/ApiService/ApiService/OneFuzzTypes/Events.cs

@@ -1,5 +1,4 @@
-using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using PoolName = System.String;
@@ -37,7 +36,7 @@ public enum EventType
     FileAdded,
     TaskHeartbeat,
     NodeHeartbeat,
-    InstanceConfigUpdated
+    InstanceConfigUpdated,
 }
 
 public abstract record BaseEvent()
@@ -50,6 +49,9 @@ public EventType GetEventType()
                 EventNodeHeartbeat _ => EventType.NodeHeartbeat,
                 EventTaskHeartbeat _ => EventType.TaskHeartbeat,
                 EventInstanceConfigUpdated _ => EventType.InstanceConfigUpdated,
+                EventCrashReported _ => EventType.CrashReported,
+                EventRegressionReported _ => EventType.RegressionReported,
+                EventFileAdded _ => EventType.FileAdded,
                 _ => throw new NotImplementedException(),
             };
 
@@ -62,6 +64,9 @@ public static Type GetTypeInfo(EventType eventType)
             EventType.NodeHeartbeat => typeof(EventNodeHeartbeat),
             EventType.InstanceConfigUpdated => typeof(EventInstanceConfigUpdated),
             EventType.TaskHeartbeat => typeof(EventTaskHeartbeat),
+            EventType.CrashReported => typeof(EventCrashReported),
+            EventType.RegressionReported => typeof(EventRegressionReported),
+            EventType.FileAdded => typeof(EventFileAdded),
             _ => throw new ArgumentException($"invalid input {eventType}"),
 
         };
@@ -249,25 +254,25 @@ PoolName PoolName
 //        NodeState state
 //        ) : BaseEvent();
 
-//    record EventCrashReported(
-//        Report Report,
-//        Container Container,
-//        [property: JsonPropertyName("filename")] String FileName,
-//        TaskConfig? TaskConfig
-//        ) : BaseEvent();
+record EventCrashReported(
+    Report Report,
+    Container Container,
+    [property: JsonPropertyName("filename")] String FileName,
+    TaskConfig? TaskConfig
+) : BaseEvent();
 
-//    record EventRegressionReported(
-//        RegressionReport RegressionReport,
-//        Container Container,
-//        [property: JsonPropertyName("filename")] String FileName,
-//        TaskConfig? TaskConfig
-//        ) : BaseEvent();
+record EventRegressionReported(
+    RegressionReport RegressionReport,
+    Container Container,
+    [property: JsonPropertyName("filename")] String FileName,
+    TaskConfig? TaskConfig
+) : BaseEvent();
 
 
-//    record EventFileAdded(
-//        Container Container,
-//        [property: JsonPropertyName("filename")] String FileName
-//        ) : BaseEvent();
+record EventFileAdded(
+    Container Container,
+    [property: JsonPropertyName("filename")] String FileName
+) : BaseEvent();
 
 
 public record EventInstanceConfigUpdated(
@@ -296,4 +301,4 @@ public override void Write(Utf8JsonWriter writer, BaseEvent value, JsonSerialize
         var eventType = value.GetType();
         JsonSerializer.Serialize(writer, value, eventType, options);
     }
-}
+}

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -1,9 +1,5 @@
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
-using System.Collections.Generic;
 using System.Text.Json.Serialization;
-
-using Container = System.String;
 using Region = System.String;
 using PoolName = System.String;
 using Endpoint = System.String;
@@ -397,32 +393,74 @@ Dictionary<string, string> Tags
 
 ) : EntityBase();
 
+public record Container(string ContainerName)
+{
+    public string ContainerName { get; } = ContainerName.All(c => char.IsLetterOrDigit(c) || c == '-') ? ContainerName : throw new ArgumentException("Container name must have only numbers, letters or dashes");
+}
+
+public record Notification(
+    DateTime? Timestamp,
+    Container Container,
+    Guid NotificationId,
+    NotificationTemplate Config
+) : EntityBase();
 
 public record BlobRef(
     string Account,
-    Container Container,
-    string Name
+    Container container,
+    string name
 );
 
-
 public record Report(
-    string? InputURL,
+    string? InputUrl,
     BlobRef? InputBlob,
-    string? Executable,
+    string Executable,
     string CrashType,
     string CrashSite,
     List<string> CallStack,
     string CallStackSha256,
     string InputSha256,
     string? AsanLog,
-    Guid TaskID,
-    Guid JobID,
+    Guid TaskId,
+    Guid JobId,
     int? ScarinessScore,
     string? ScarinessDescription,
-    List<string> MinimizedStack,
+    List<string>? MinimizedStack,
     string? MinimizedStackSha256,
-    List<string> MinimizedStackFunctionNames,
+    List<string>? MinimizedStackFunctionNames,
     string? MinimizedStackFunctionNamesSha256,
-    List<string> MinimizedStackFunctionLines,
+    List<string>? MinimizedStackFunctionLines,
     string? MinimizedStackFunctionLinesSha256
 );
+
+public record NoReproReport(
+    string InputSha,
+    BlobRef? InputBlob,
+    string? Executable,
+    Guid TaskId,
+    Guid JobId,
+    int Tries,
+    string? Error
+);
+
+public record CrashTestResult(
+    Report? CrashReport,
+    NoReproReport? NoReproReport
+);
+
+public record RegressionReport(
+    CrashTestResult CrashTestResult,
+    CrashTestResult? OriginalCrashTestResult
+);
+
+public record NotificationTemplate(
+    AdoTemplate? AdoTemplate,
+    TeamsTemplate? TeamsTemplate,
+    GithubIssuesTemplate? GithubIssuesTemplate
+);
+
+public record AdoTemplate();
+
+public record TeamsTemplate();
+
+public record GithubIssuesTemplate();

src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs

@@ -1,6 +1,4 @@
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
-using System.Collections.Generic;
 using System.Text.Json.Serialization;
 
 namespace Microsoft.OneFuzz.Service;

src/ApiService/ApiService/Program.cs

@@ -1,8 +1,10 @@
 // to avoid collision with Task in model.cs
 global using Async = System.Threading.Tasks;
 
-using System;
-using System.Collections.Generic;
+global using System;
+global using System.Collections.Generic;
+global using System.Linq;
+
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Azure.Functions.Worker.Middleware;
@@ -76,6 +78,9 @@ public static void Main()
             .AddScoped<IProxyOperations, ProxyOperations>()
             .AddScoped<IConfigOperations, ConfigOperations>()
             .AddScoped<IScalesetOperations, ScalesetOperations>()
+            .AddScoped<IContainers, Containers>()
+            .AddScoped<IReports, Reports>()
+            .AddScoped<INotificationOperations, NotificationOperations>()
 
         //TODO: move out expensive resources into separate class, and add those as Singleton
         // ArmClient, Table Client(s), Queue Client(s), HttpClient, etc.