Merge branch 'main' into user/noharper/report-empty-coverage

microsoft · Oct 30, 2023 · 13ce606 · 13ce606
2 parents 7b8ebcb + d50fd48
commit 13ce606
Show file tree

Hide file tree

Showing 18 changed files with 155 additions and 56 deletions.
diff --git a/src/ApiService/ApiService/Functions/Containers.cs b/src/ApiService/ApiService/Functions/Containers.cs
@@ -16,11 +16,12 @@ public ContainersFunction(ILogger<ContainersFunction> logger, IOnefuzzContext co
 
     [Function("Containers")]
     [Authorize(Allow.User)]
-    public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", "POST", "DELETE")] HttpRequestData req)
+    public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", "POST", "PATCH", "DELETE")] HttpRequestData req)
         => req.Method switch {
             "GET" => Get(req),
             "POST" => Post(req),
             "DELETE" => Delete(req),
+            "PATCH" => Patch(req),
             _ => throw new NotSupportedException(),
         };
 
@@ -108,4 +109,21 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
                 SasUrl: sas,
                 Metadata: post.Metadata));
     }
+
+    private async Async.Task<HttpResponseData> Patch(HttpRequestData req) {
+        var request = await RequestHandling.ParseRequest<ContainerUpdate>(req);
+        if (!request.IsOk) {
+            return await _context.RequestHandling.NotOk(req, request.ErrorV, context: "container update");
+        }
+
+        var toUpdate = request.OkV;
+        _logger.LogInformation("updating {ContainerName}", toUpdate.Name);
+        var updated = await _context.Containers.CreateOrUpdateContainerTag(toUpdate.Name, StorageType.Corpus, toUpdate.Metadata.ToDictionary(x => x.Key, x => x.Value));
+
+        if (!updated.IsOk) {
+            return await _context.RequestHandling.NotOk(req, updated.ErrorV, "container update");
+        }
+
+        return await RequestHandling.Ok(req, new ContainerInfoBase(toUpdate.Name, toUpdate.Metadata));
+    }
 }
diff --git a/src/ApiService/ApiService/Functions/QueueFileChanges.cs b/src/ApiService/ApiService/Functions/QueueFileChanges.cs
@@ -2,6 +2,8 @@
 using System.Text.Json.Nodes;
 using System.Threading.Tasks;
 using Azure.Core;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Extensions.Logging;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
@@ -60,7 +62,11 @@ public async Async.Task Run(
         try {
             var result = await FileAdded(storageAccount, fileChangeEvent);
             if (!result.IsOk) {
-                await RequeueMessage(msg, result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED ? TimeSpan.FromDays(1) : null);
+                if (result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
+                    await RequeueMessage(msg, TimeSpan.FromDays(1), incrementDequeueCount: false);
+                } else {
+                    await RequeueMessage(msg);
+                }
             }
         } catch (Exception e) {
             _log.LogError(e, "File Added failed");
@@ -83,21 +89,26 @@ private async Async.Task<OneFuzzResultVoid> FileAdded(ResourceIdentifier storage
 
         _log.LogInformation("file added : {Container} - {Path}", container.String, path);
 
+        var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
+        var containerClient = account.GetBlobContainerClient(container.String);
+        var containerProps = await containerClient.GetPropertiesAsync();
+
+        if (_context.NotificationOperations.ShouldPauseNotificationsForContainer(containerProps.Value.Metadata)) {
+            return Error.Create(ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED, $"container {container} has a metadata tag set to pause notifications processing");
+        }
+
         var (_, result) = await (
-            ApplyRetentionPolicy(storageAccount, container, path),
+            ApplyRetentionPolicy(containerClient, containerProps, path),
             _notificationOperations.NewFiles(container, path));
 
         return result;
     }
 
-    private async Async.Task<bool> ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) {
+    private async Async.Task<bool> ApplyRetentionPolicy(BlobContainerClient containerClient, BlobContainerProperties containerProps, string path) {
         if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableContainerRetentionPolicies)) {
             // default retention period can be applied to the container
             // if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one
-            var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
-            var containerClient = account.GetBlobContainerClient(container.String);
-            var containerProps = await containerClient.GetPropertiesAsync();
-            var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Value.Metadata);
+            var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Metadata);
             if (!retentionPeriod.IsOk) {
                 _log.LogError("invalid retention period: {Error}", retentionPeriod.ErrorV);
             } else if (retentionPeriod.OkV is TimeSpan period) {
@@ -116,7 +127,7 @@ private async Async.Task<bool> ApplyRetentionPolicy(ResourceIdentifier storageAc
         return false;
     }
 
-    private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {
+    private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null, bool incrementDequeueCount = true) {
         var json = JsonNode.Parse(msg);
 
         // Messages that are 'manually' requeued by us as opposed to being requeued by the azure functions runtime
@@ -135,7 +146,9 @@ await _context.Queue.QueueObject(
                 StorageType.Config)
                 .IgnoreResult();
         } else {
-            json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1;
+            if (incrementDequeueCount) {
+                json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1;
+            }
             await _context.Queue.QueueObject(
                 QueueFileChangesQueueName,
                 json,

diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs
@@ -53,6 +53,9 @@ public enum ErrorCode {
     INVALID_RETENTION_PERIOD = 497,
     INVALID_CLI_VERSION = 498,
     TRANSIENT_NOTIFICATION_FAILURE = 499,
+
+    FAILED_CONTAINER_PROPERTIES_ACCESS = 500,
+    FAILED_SAVING_CONTAINER_METADATA = 501,
     // NB: if you update this enum, also update enums.py
 }
 

diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs
@@ -128,6 +128,11 @@ public record ContainerDelete(
     IDictionary<string, string>? Metadata = null
 ) : BaseRequest;
 
+public record ContainerUpdate(
+    [property: Required] Container Name,
+    [property: Required] IDictionary<string, string> Metadata
+) : BaseRequest;
+
 public record NotificationCreate(
     [property: Required] Container Container,
     [property: Required] bool ReplaceExisting,

diff --git a/src/ApiService/ApiService/onefuzzlib/Containers.cs b/src/ApiService/ApiService/onefuzzlib/Containers.cs
@@ -1,5 +1,6 @@
 using System.IO;
 using System.IO.Compression;
+using System.Text.Json;
 using System.Threading.Tasks;
 using ApiService.OneFuzzLib.Orm;
 using Azure;
@@ -41,6 +42,8 @@ public interface IContainers {
     public Async.Task<OneFuzzResultVoid> DownloadAsZip(Container container, StorageType storageType, Stream stream, string? prefix = null);
 
     public Async.Task DeleteAllExpiredBlobs();
+
+    public Async.Task<OneFuzzResultVoid> CreateOrUpdateContainerTag(Container container, StorageType storageType, Dictionary<string, string> tags);
 }
 
 public class Containers : Orm<ContainerInformation>, IContainers {
@@ -448,4 +451,29 @@ private async Async.Task DeleteExpiredBlobsForAccount(ResourceIdentifier storage
             }
         }
     }
+
+    public async Task<OneFuzzResultVoid> CreateOrUpdateContainerTag(Container container, StorageType storageType, Dictionary<string, string> tags) {
+        var client = await FindContainer(container, storageType);
+        if (client is null || !await client.ExistsAsync()) {
+            return Error.Create(ErrorCode.INVALID_CONTAINER, $"Could not find container {container} in {storageType}");
+        }
+
+        var metadataRequest = await client.GetPropertiesAsync();
+        if (metadataRequest is null || metadataRequest.GetRawResponse().IsError) {
+            return Error.Create(ErrorCode.FAILED_CONTAINER_PROPERTIES_ACCESS, $"Could not access container properties for container: {container} in {storageType}");
+        }
+
+        var metadata = metadataRequest.Value.Metadata ?? new Dictionary<string, string>();
+
+        foreach (var kvp in tags) {
+            metadata[kvp.Key] = kvp.Value;
+        }
+
+        var saveMetadataRequest = await client.SetMetadataAsync(metadata);
+        if (saveMetadataRequest is null || saveMetadataRequest.GetRawResponse().IsError) {
+            return Error.Create(ErrorCode.FAILED_SAVING_CONTAINER_METADATA, $"Could not save metadata to container: {container} in {storageType}. Metadata: {JsonSerializer.Serialize(metadata)}");
+        }
+
+        return OneFuzzResultVoid.Ok;
+    }
 }
diff --git a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs
@@ -11,12 +11,11 @@ public interface INotificationOperations : IOrm<Notification> {
     Async.Task<OneFuzzResult<Notification>> Create(Container container, NotificationTemplate config, bool replaceExisting);
     Async.Task<Notification?> GetNotification(Guid notifificationId);
 
-    System.Threading.Tasks.Task<OneFuzzResultVoid> TriggerNotification(Container container,
-        Notification notification, IReport? reportOrRegression);
+    System.Threading.Tasks.Task<OneFuzzResultVoid> TriggerNotification(Container container, Notification notification, IReport? reportOrRegression);
+    bool ShouldPauseNotificationsForContainer(IDictionary<string, string> containerMetadata);
 }
 
 public class NotificationOperations : Orm<Notification>, INotificationOperations {
-
     public NotificationOperations(ILogger<NotificationOperations> log, IOnefuzzContext context)
         : base(log, context) {
 
@@ -190,4 +189,7 @@ private async Async.Task<NotificationTemplate> HideSecrets(NotificationTemplate
     public async Async.Task<Notification?> GetNotification(Guid notifificationId) {
         return await SearchByPartitionKeys(new[] { notifificationId.ToString() }).SingleOrDefaultAsync();
     }
+
+    private const string PAUSE_NOTIFICATIONS_TAG = "pauseNotifications";
+    public bool ShouldPauseNotificationsForContainer(IDictionary<string, string> containerMetadata) => containerMetadata.ContainsKey(PAUSE_NOTIFICATIONS_TAG) && containerMetadata[PAUSE_NOTIFICATIONS_TAG] == "true";
 }
diff --git a/src/ApiService/ApiService/onefuzzlib/Reports.cs b/src/ApiService/ApiService/onefuzzlib/Reports.cs
@@ -67,7 +67,6 @@ public Reports(ILogger<Reports> log, IContainers containers) {
     }
 
     private static T? TryDeserialize<T>(string content) where T : class {
-
         try {
             return JsonSerializer.Deserialize<T>(content, EntityConverter.GetJsonSerializerOptions());
         } catch (JsonException) {

diff --git a/src/agent/onefuzz-task/src/tasks/report/crash_report.rs b/src/agent/onefuzz-task/src/tasks/report/crash_report.rs
@@ -6,7 +6,7 @@ use onefuzz::{blob::BlobUrl, monitor::DirectoryMonitor, syncdir::SyncedDir};
 use onefuzz_result::job_result::{JobResultData, JobResultSender, TaskJobResultClient};
 use onefuzz_telemetry::{
     Event::{
-        new_report, new_unable_to_reproduce, new_unique_report, regression_report,
+        crash_reported, new_report, new_unable_to_reproduce, new_unique_report, regression_report,
         regression_unable_to_reproduce,
     },
     EventData,
@@ -166,6 +166,8 @@ impl CrashTestResult {
         match self {
             Self::CrashReport(report) => {
                 // Use SHA-256 of call stack as dedupe key.
+                event!(crash_reported; EventData::Path = report.unique_blob_name());
+                metric!(crash_reported; 1.0; EventData::Path = report.unique_blob_name());
                 if let Some(jr_client) = jr_client {
                     let _ = jr_client
                         .send_direct(

diff --git a/src/agent/onefuzz-telemetry/src/lib.rs b/src/agent/onefuzz-telemetry/src/lib.rs
@@ -83,6 +83,7 @@ pub enum Event {
     runtime_stats,
     new_report,
     new_unique_report,
+    crash_reported,
     new_unable_to_reproduce,
     regression_report,
     regression_unable_to_reproduce,
@@ -101,6 +102,7 @@ impl Event {
             Self::runtime_stats => "runtime_stats",
             Self::new_report => "new_report",
             Self::new_unique_report => "new_unique_report",
+            Self::crash_reported => "crash_reported",
             Self::new_unable_to_reproduce => "new_unable_to_reproduce",
             Self::regression_report => "regression_report",
             Self::regression_unable_to_reproduce => "regression_unable_to_reproduce",

diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py
@@ -486,6 +486,17 @@ def delete(self, name: str) -> responses.BoolResult:
             "DELETE", responses.BoolResult, data=requests.ContainerDelete(name=name)
         )
 
+    def update(
+        self, name: str, metadata: Dict[str, str]
+    ) -> responses.ContainerInfoBase:
+        """Update a container's metadata"""
+        self.logger.debug("update container: %s", name)
+        return self._req_model(
+            "PATCH",
+            responses.ContainerInfoBase,
+            data=requests.ContainerUpdate(name=name, metadata=metadata),
+        )
+
     def list(self) -> List[responses.ContainerInfoBase]:
         """Get a list of containers"""
         self.logger.debug("list containers")

diff --git a/src/deployment/azuredeploy.bicep b/src/deployment/azuredeploy.bicep
@@ -156,7 +156,6 @@ module storage 'bicep-templates/storageAccounts.bicep' = {
   params: {
     location: location
     owner: owner
-    signedExpiry: signedExpiry
   }
 }
 
@@ -172,6 +171,7 @@ module autoscaleSettings 'bicep-templates/autoscale-settings.bicep' = {
   }
 }
 
+
 module eventGrid 'bicep-templates/event-grid.bicep' = {
   name: 'event-grid'
   params: {
@@ -227,8 +227,8 @@ module function 'bicep-templates/function.bicep' = {
   params: {
     name: name
     linux_fx_version: 'DOTNET-ISOLATED|7.0'
-
-    app_logs_sas_url: storage.outputs.FuncSasUrlBlobAppLogs
+    signedExpiry: signedExpiry
+    logs_storage: storage.outputs.FuncName
     app_func_audiences: app_func_audiences
     app_func_issuer: app_func_issuer
     client_id: clientId
@@ -241,6 +241,9 @@ module function 'bicep-templates/function.bicep' = {
     use_windows: true
     enable_remote_debugging: enable_remote_debugging
   }
+  dependsOn:[
+    storage
+  ]
 }
 
 module functionSettings 'bicep-templates/function-settings.bicep' = {
@@ -254,8 +257,9 @@ module functionSettings 'bicep-templates/function-settings.bicep' = {
     app_insights_app_id: operationalInsights.outputs.appInsightsAppId
     app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey
     client_secret: clientSecret
-    signal_r_connection_string: signalR.outputs.connectionString
-    func_sas_url: storage.outputs.FuncSasUrl
+
+    signalRName: signalR.outputs.signalRName
+    funcStorageName: storage.outputs.FuncName
     func_storage_resource_id: storage.outputs.FuncId
     fuzz_storage_resource_id: storage.outputs.FuzzId
     keyvault_name: keyVaultName
@@ -269,16 +273,18 @@ module functionSettings 'bicep-templates/function-settings.bicep' = {
   }
   dependsOn: [
     function
+    storage
+    signalR
   ]
 }
 
 output fuzz_storage string = storage.outputs.FuzzId
 output fuzz_name string = storage.outputs.FuzzName
-output fuzz_key string = storage.outputs.FuzzKey
+
 
 output func_storage string = storage.outputs.FuncId
 output func_name string = storage.outputs.FuncName
-output func_key string = storage.outputs.FuncKey
+
 
 output scaleset_identity string = scaleset_identity
 output tenant_id string = tenantId

diff --git a/src/deployment/bicep-templates/function-settings.bicep b/src/deployment/bicep-templates/function-settings.bicep
@@ -5,17 +5,11 @@ param app_insights_app_id string
 @secure()
 param app_insights_key string
 
-@secure()
-param func_sas_url string
-
 param cli_app_id string
 param authority string
 param tenant_domain string
 param multi_tenant_domain string
 
-@secure()
-param signal_r_connection_string string
-
 param app_config_endpoint string
 
 param func_storage_resource_id string
@@ -33,8 +27,21 @@ param functions_extension_version string
 
 param enable_profiler bool
 
+param signalRName string
+param funcStorageName string
+
 var telemetry = 'd7a73cf4-5a1a-4030-85e1-e5b25867e45a'
 
+
+resource signal_r 'Microsoft.SignalRService/signalR@2021-10-01' existing = {
+ name: signalRName
+}
+
+
+resource funcStorage 'Microsoft.Storage/storageAccounts@2021-08-01' existing = {
+  name: funcStorageName
+}
+
 resource function 'Microsoft.Web/sites@2021-02-01' existing = {
   name: name
 }
@@ -44,6 +51,7 @@ var enable_profilers = enable_profiler ? {
   DiagnosticServices_EXTENSION_VERSION: '~3'
 } : {}
 
+var func_key = funcStorage.listKeys().keys[0].value
 resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = {
   parent: function
   name: 'appsettings'
@@ -54,13 +62,13 @@ resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = {
       APPINSIGHTS_INSTRUMENTATIONKEY: app_insights_key
       APPINSIGHTS_APPID: app_insights_app_id
       ONEFUZZ_TELEMETRY: telemetry
-      AzureWebJobsStorage: func_sas_url
+      AzureWebJobsStorage: 'DefaultEndpointsProtocol=https;AccountName=${funcStorage.name};AccountKey=${func_key};EndpointSuffix=core.windows.net'
       CLI_APP_ID: cli_app_id
       AUTHORITY: authority
       TENANT_DOMAIN: tenant_domain
       MULTI_TENANT_DOMAIN: multi_tenant_domain
       AzureWebJobsDisableHomepage: 'true'
-      AzureSignalRConnectionString: signal_r_connection_string
+      AzureSignalRConnectionString: signal_r.listKeys().primaryConnectionString
       AzureSignalRServiceTransportType: 'Transient'
       APPCONFIGURATION_ENDPOINT: app_config_endpoint
       ONEFUZZ_INSTANCE_NAME: instance_name