Add extra_output container, rename extra container

docs/command-replacements.md

@@ -26,7 +26,8 @@ The following values are replaced with the specific values at runtime.
 * `{crashes_container}`: Container name for the `crashes` container
 * `{microsoft_telemetry_key}`: Application Insights key used for collecting [non-attributable telemetry](telemetry.md) to improve OneFuzz.
 * `{instance_telemetry_key}`: Application Insights key used for private, instance-owned telemetry and logging (See [OneFuzz Telemetry](telemetry.md).
-* `{extra_dir}`: Path to the optionally provided `extra` directory
+* `{extra_setup_dir}`: Path to the optionally provided `extra_setup` directory 
+* `{extra_output_dir}`: Path to the optionally provided `extra_output` directory
 
 ## Example
 

docs/webhook_events.md

@@ -152,7 +152,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },
@@ -2023,7 +2024,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },
@@ -2959,7 +2961,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },
@@ -3441,7 +3444,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },
@@ -3976,7 +3980,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },
@@ -4425,7 +4430,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },
@@ -4901,7 +4907,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },
@@ -5507,7 +5514,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "unique_reports",
                 "regression_reports",
                 "logs",
-                "extra"
+                "extra_setup",
+                "extra_output"
             ],
             "title": "ContainerType"
         },

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -107,7 +107,8 @@ public enum ContainerType {
     UniqueReports,
     RegressionReports,
     Logs,
-    Extra
+    ExtraSetup,
+    ExtraOutput,
 }
 
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -917,7 +917,7 @@ public record VmDefinition(
     long Value
 );
 
-public record TaskDefinition(
+public readonly record struct TaskDefinition(
     TaskFeature[] Features,
     VmDefinition Vm,
     ContainerDefinition[] Containers,
@@ -927,42 +927,36 @@ public record TaskDefinition(
 public record WorkSet(
     bool Reboot,
     Uri SetupUrl,
-    Uri? ExtraUrl,
+    Uri? ExtraSetupUrl,
     bool Script,
     List<WorkUnit> WorkUnits
 );
 
-
-
-
-
-public record ContainerDefinition(
+public readonly record struct ContainerDefinition(
     ContainerType Type,
     Compare Compare,
     long Value,
     ContainerPermission Permissions);
 
-
 // TODO: service shouldn't pass SyncedDir, but just the url and let the agent
 // come up with paths
-public record SyncedDir(string Path, Uri Url);
-
+public readonly record struct SyncedDir(string Path, Uri Url);
 
 [JsonConverter(typeof(ContainerDefConverter))]
 public interface IContainerDef { }
 public record SingleContainer(SyncedDir SyncedDir) : IContainerDef;
-public record MultipleContainer(List<SyncedDir> SyncedDirs) : IContainerDef;
+public record MultipleContainer(IReadOnlyList<SyncedDir> SyncedDirs) : IContainerDef;
 
 
 public class ContainerDefConverter : JsonConverter<IContainerDef> {
     public override IContainerDef? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
         if (reader.TokenType == JsonTokenType.StartObject) {
             var result = (SyncedDir?)JsonSerializer.Deserialize(ref reader, typeof(SyncedDir), options);
-            if (result is null) {
-                return null;
+            if (result is SyncedDir sd) {
+                return new SingleContainer(sd);
             }
 
-            return new SingleContainer(result);
+            return null;
         }
 
         if (reader.TokenType == JsonTokenType.StartArray) {
@@ -1057,8 +1051,8 @@ Dictionary<string, string> Tags
     public IContainerDef? UniqueInputs { get; set; }
     public IContainerDef? UniqueReports { get; set; }
     public IContainerDef? RegressionReports { get; set; }
-    public IContainerDef? Extra { get; set; }
-
+    public IContainerDef? ExtraSetup { get; set; }
+    public IContainerDef? ExtraOutput { get; set; }
 }
 
 public record NodeCommandEnvelope(

src/ApiService/ApiService/onefuzzlib/Config.cs

@@ -75,75 +75,82 @@ private static BlobContainerSasPermissions ConvertPermissions(ContainerPermissio
         );
 
         if (definition.MonitorQueue != null) {
-            config.inputQueue = await _queue.GetQueueSas(task.TaskId.ToString(), StorageType.Corpus, QueueSasPermissions.Add | QueueSasPermissions.Read | QueueSasPermissions.Update | QueueSasPermissions.Process);
-        }
-
-        var containersByType = definition.Containers.Where(c => c.Type != ContainerType.Setup && task.Config.Containers != null)
-            .ToAsyncEnumerable()
-            .SelectAwait(async countainerDef => {
-                var containers = await
-                    task.Config.Containers!
-                        .Where(c => c.Type == countainerDef.Type).Select(container => (countainerDef, container))
-                        .Where(x => x.container != null)
-                        .ToAsyncEnumerable()
-                        .SelectAwait(async (x, i) =>
-                                new SyncedDir(
-                                    string.Join("_", "task", x.Item1.Type.ToString().ToLower(), i),
-                                    await _containers.GetContainerSasUrl(x.Item2.Name, StorageType.Corpus, ConvertPermissions(x.Item1.Permissions)))
-                        ).ToListAsync();
-                return (countainerDef, containers);
-            }
-                );
-
-        await foreach (var data in containersByType) {
-
-            if (!data.containers.Any()) {
-                continue;
-            }
+            config.inputQueue = await _queue.GetQueueSas(task.TaskId.ToString(), StorageType.Corpus, QueueSasPermissions.All);
+        }
+
+        if (task.Config.Containers is not null) {
+            var containersByType =
+                await Async.Task.WhenAll(
+                    definition.Containers
+                    .Where(c => c.Type is not ContainerType.Setup)
+                    .Select(async countainerDef => {
+                        var syncedDirs =
+                            await Async.Task.WhenAll(
+                                task.Config.Containers
+                                .Where(c => c.Type == countainerDef.Type)
+                                .Select(async (container, i) =>
+                                    new SyncedDir(
+                                        string.Join("_", "task", countainerDef.Type.ToString().ToLower(), i),
+                                        await _containers.GetContainerSasUrl(container.Name, StorageType.Corpus, ConvertPermissions(countainerDef.Permissions)))
+                                ));
+
+                        return (countainerDef, syncedDirs);
+                    }));
+
+            foreach (var (containerDef, syncedDirs) in containersByType) {
+                if (!syncedDirs.Any()) {
+                    continue;
+                }
 
-            IContainerDef def = data.countainerDef switch {
-                ContainerDefinition { Compare: Compare.Equal, Value: 1 } or
-                ContainerDefinition { Compare: Compare.AtMost, Value: 1 } when data.containers.Count == 1 => new SingleContainer(data.containers[0]),
-                _ => new MultipleContainer(data.containers)
-            };
-
-            switch (data.countainerDef.Type) {
-                case ContainerType.Analysis:
-                    config.Analysis = def;
-                    break;
-                case ContainerType.Coverage:
-                    config.Coverage = def;
-                    break;
-                case ContainerType.Crashes:
-                    config.Crashes = def;
-                    break;
-                case ContainerType.Inputs:
-                    config.Inputs = def;
-                    break;
-                case ContainerType.NoRepro:
-                    config.NoRepro = def;
-                    break;
-                case ContainerType.ReadonlyInputs:
-                    config.ReadonlyInputs = def;
-                    break;
-                case ContainerType.Reports:
-                    config.Reports = def;
-                    break;
-                case ContainerType.Tools:
-                    config.Tools = def;
-                    break;
-                case ContainerType.UniqueInputs:
-                    config.UniqueInputs = def;
-                    break;
-                case ContainerType.UniqueReports:
-                    config.UniqueReports = def;
-                    break;
-                case ContainerType.RegressionReports:
-                    config.RegressionReports = def;
-                    break;
-                case ContainerType.Extra:
-                    config.Extra = def;
-                    break;
+                IContainerDef def = containerDef switch {
+                    ContainerDefinition { Compare: Compare.Equal or Compare.AtMost, Value: 1 }
+                        when syncedDirs is [var syncedDir] => new SingleContainer(syncedDir),
+                    _ => new MultipleContainer(syncedDirs)
+                };
+
+                switch (containerDef.Type) {
+                    case ContainerType.Analysis:
+                        config.Analysis = def;
+                        break;
+                    case ContainerType.Coverage:
+                        config.Coverage = def;
+                        break;
+                    case ContainerType.Crashes:
+                        config.Crashes = def;
+                        break;
+                    case ContainerType.Inputs:
+                        config.Inputs = def;
+                        break;
+                    case ContainerType.NoRepro:
+                        config.NoRepro = def;
+                        break;
+                    case ContainerType.ReadonlyInputs:
+                        config.ReadonlyInputs = def;
+                        break;
+                    case ContainerType.Reports:
+                        config.Reports = def;
+                        break;
+                    case ContainerType.Tools:
+                        config.Tools = def;
+                        break;
+                    case ContainerType.UniqueInputs:
+                        config.UniqueInputs = def;
+                        break;
+                    case ContainerType.UniqueReports:
+                        config.UniqueReports = def;
+                        break;
+                    case ContainerType.RegressionReports:
+                        config.RegressionReports = def;
+                        break;
+                    case ContainerType.ExtraSetup:
+                        config.ExtraSetup = def;
+                        break;
+                    case ContainerType.ExtraOutput:
+                        config.ExtraOutput = def;
+                        break;
+                    default:
+                        throw new InvalidDataException($"unknown container type: {containerDef.Type}");
+                }
             }
         }