Release 8.6.3 (Hotfix)

CHANGELOG.md

@@ -7,6 +7,56 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 8.6.3
+
+### Fixed
+
+* Service: Fixed another duplicate Azure DevOps work item creation case by handling `Microsoft.VSTS.Common.ResolvedReason` field when present [#3383](https://github.com/microsoft/onefuzz/pull/3383)
+
+## 8.6.2
+
+### Fixed
+
+* Agent: Fixed tasks hanging when shutting down by forcefully shutting down the runtime before exiting the main task [#3378](https://github.com/microsoft/onefuzz/pull/3378)
+* Service: Refactored Azure DevOps template rendering to fix duplicate bugs being filed due to title truncation and added several validation tests in this area [#3370](https://github.com/microsoft/onefuzz/pull/3370)
+
+## 8.6.1
+
+### Added
+
+* Service: Added feature flag to toggle Azure DevOps work item processing [#3353](https://github.com/microsoft/onefuzz/pull/3353)
+* Service: Requeue Azure DevOps notifications when the feature flag for work item processing is set to 'disabled' [#3358](https://github.com/microsoft/onefuzz/pull/3358)
+
+## 8.6.0
+
+### Added
+
+* Agent: Implemented `debuginfo` caching [#3280](https://github.com/microsoft/onefuzz/pull/3280)
+
+### Changed
+
+* Agent: Limit azcopy copy buffer to 512MB of RAM as the default maximum [#3293](https://github.com/microsoft/onefuzz/pull/3293)
+* Agent: Define local fuzzing tasks relationships through new templating model [#3117](https://github.com/microsoft/onefuzz/pull/3117)
+* Deployment: Replaced `--upgrade` flag with `--skip_aad_setup` flag in the deploy.py setup script [#3345](https://github.com/microsoft/onefuzz/pull/3345)
+* Service: Make `ServiceConfiguration` eagerly evaluated [#3136](https://github.com/microsoft/onefuzz/pull/3136)
+* Service: Improved `TimerRetention` performance through several UPN changes & fixes [#3289](https://github.com/microsoft/onefuzz/pull/3289)
+
+### Fixed
+
+* Agent: Fixed resolution of sibling .NET DLLs [#3325](https://github.com/microsoft/onefuzz/pull/3325)
+* Agent/Service: Bumped several C# and Rust dependencies [#3319](https://github.com/microsoft/onefuzz/pull/3319), [#3320](https://github.com/microsoft/onefuzz/pull/3320), [#3317](https://github.com/microsoft/onefuzz/pull/3317), [#3297](https://github.com/microsoft/onefuzz/pull/3297), [#3301](https://github.com/microsoft/onefuzz/pull/3301), [#3291](https://github.com/microsoft/onefuzz/pull/3291), [#3195](https://github.com/microsoft/onefuzz/pull/3195), [#3328](https://github.com/microsoft/onefuzz/pull/3328)
+* CLI: Look for azcopy.exe in environment variable `AZCOPY` and determine if it's actually referencing a directory [#3344](https://github.com/microsoft/onefuzz/pull/3344)
+* CLI: Updated `repro get_files` to handle regression reports [#3340](https://github.com/microsoft/onefuzz/pull/3340)
+* CLI: Fixed missing `target_timeout` setting in the Libfuzzer basic template [#3334](https://github.com/microsoft/onefuzz/pull/3334)
+* CLI: Fixed false 'missing' dependency warning [#3331](https://github.com/microsoft/onefuzz/pull/3331)
+* CLI: Fixed the `debug notification test_template` command expecting a `task_id`  [#3308](https://github.com/microsoft/onefuzz/pull/3308)
+* Deployment: Update App Registration redirect URIs if deployment uses a custom domain [#3341](https://github.com/microsoft/onefuzz/pull/3341)
+* Service: Fixed links in bugs filed from regression reports by populating `InputBlob` when possible [#3342](https://github.com/microsoft/onefuzz/pull/3342)
+* Service: Fixed several storage issues to improve platform performance and reduce spurious `404`s [#3313](https://github.com/microsoft/onefuzz/pull/3313)
+* Service: Added extra logging when `System.Title` is too long [#3332](https://github.com/microsoft/onefuzz/pull/3332)
+* Service: Render `System.Title` before trying to trim it to the max allowed size [#3329](https://github.com/microsoft/onefuzz/pull/3329)
+* Service: Differentiate `INVALID_JOB` and `INVALID_TASK` error codes [#3318](https://github.com/microsoft/onefuzz/pull/3318)
+
 ## 8.5.0
 
 ### Added

CURRENT_VERSION

@@ -1 +1 @@
-8.5.0
+8.6.3

src/ApiService/ApiService/FeatureFlags.cs

@@ -7,4 +7,5 @@ public static class FeatureFlagConstants {
     public const string EnableCustomMetricTelemetry = "EnableCustomMetricTelemetry";
     public const string EnableBlobRetentionPolicy = "EnableBlobRetentionPolicy";
     public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention";
+    public const string EnableWorkItemCreation = "EnableWorkItemCreation";
 }

src/ApiService/ApiService/Functions/QueueFileChanges.cs

@@ -61,14 +61,17 @@ public async Async.Task Run(
             // requeuing ourselves because azure functions doesn't support retry policies
             // for queue based functions.
 
-            await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
+                await RequeueMessage(msg, TimeSpan.FromDays(1));
+            }
         } catch (Exception e) {
             _log.LogError(e, "File Added failed");
             await RequeueMessage(msg);
         }
     }
 
-    private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
+    private async Async.Task<OneFuzzResultVoid> FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
         var data = fileChangeEvent.RootElement.GetProperty("data");
         var url = data.GetProperty("url").GetString()!;
         var parts = url.Split("/").Skip(3).ToList();
@@ -77,10 +80,10 @@ private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetr
         var path = string.Join('/', parts.Skip(1));
 
         _log.LogInformation("file added : {Container} - {Path}", container, path);
-        await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
+        return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
     }
 
-    private async Async.Task RequeueMessage(string msg) {
+    private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {
         var json = JsonNode.Parse(msg);
 
         // Messages that are 'manually' requeued by us as opposed to being requeued by the azure functions runtime
@@ -103,7 +106,7 @@ await _context.Queue.QueueObject(
             queueName,
             json,
             StorageType.Config,
-            CalculateExponentialBackoff(newCustomDequeueCount))
+            visibilityTimeout ?? CalculateExponentialBackoff(newCustomDequeueCount))
             .IgnoreResult();
     }
 

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -47,6 +47,7 @@ public enum ErrorCode {
     ADO_VALIDATION_UNEXPECTED_HTTP_EXCEPTION = 490,
     ADO_VALIDATION_UNEXPECTED_ERROR = 491,
     ADO_VALIDATION_MISSING_PAT_SCOPES = 492,
+    ADO_WORKITEM_PROCESSING_DISABLED = 494,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -672,6 +672,17 @@ public async Task<OneFuzzResultVoid> Validate() {
     }
 }
 
+public record RenderedAdoTemplate(
+    Uri BaseUrl,
+    SecretData<string> AuthToken,
+    string Project,
+    string Type,
+    List<string> UniqueFields,
+    Dictionary<string, string> AdoFields,
+    ADODuplicateTemplate OnDuplicate,
+    string? Comment = null
+    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment);
+
 public record TeamsTemplate(SecretData<string> Url) : NotificationTemplate {
     public Task<OneFuzzResultVoid> Validate() {
         // The only way we can validate in the current state is to send a test webhook

src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs

@@ -31,7 +31,7 @@ public async Task<HttpResponseData> NewFiles([HttpTrigger(AuthorizationLevel.Ano
             var fileName = query["fileName"];
             var isLastRetryAttempt = UriExtension.GetBool("isLastRetryAttempt", query, true);
 
-            await _notificationOps.NewFiles(Container.Parse(container), fileName, isLastRetryAttempt);
+            _ = await _notificationOps.NewFiles(Container.Parse(container), fileName, isLastRetryAttempt);
             var resp = req.CreateResponse(HttpStatusCode.OK);
             return resp;
         }

src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs

@@ -5,7 +5,7 @@
 namespace Microsoft.OneFuzz.Service;
 
 public interface INotificationOperations : IOrm<Notification> {
-    Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt);
+    Async.Task<OneFuzzResultVoid> NewFiles(Container container, string filename, bool isLastRetryAttempt);
     IAsyncEnumerable<Notification> GetNotifications(Container container);
     IAsyncEnumerable<(Task, IEnumerable<Container>)> GetQueueTasks();
     Async.Task<OneFuzzResult<Notification>> Create(Container container, NotificationTemplate config, bool replaceExisting);
@@ -21,10 +21,12 @@ public NotificationOperations(ILogger<NotificationOperations> log, IOnefuzzConte
         : base(log, context) {
 
     }
-    public async Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt) {
+    public async Async.Task<OneFuzzResultVoid> NewFiles(Container container, string filename, bool isLastRetryAttempt) {
+        var result = OneFuzzResultVoid.Ok;
+
         // We don't want to store file added events for the events container because that causes an infinite loop
         if (container == WellKnownContainers.Events) {
-            return;
+            return result;
         }
 
         var notifications = GetNotifications(container);
@@ -38,7 +40,10 @@ public async Async.Task NewFiles(Container container, string filename, bool isLa
                 }
 
                 done.Add(notification.Config);
-                _ = await TriggerNotification(container, notification, reportOrRegression, isLastRetryAttempt);
+                var notificationResult = await TriggerNotification(container, notification, reportOrRegression, isLastRetryAttempt);
+                if (result.IsOk && !notificationResult.IsOk) {
+                    result = notificationResult;
+                }
             }
         }
 
@@ -77,6 +82,8 @@ public async Async.Task NewFiles(Container container, string filename, bool isLa
         } else {
             await _context.Events.SendEvent(new EventFileAdded(container, filename));
         }
+
+        return result;
     }
 
     public async System.Threading.Tasks.Task<OneFuzzResultVoid> TriggerNotification(Container container,
@@ -87,8 +94,12 @@ await _context.Teams.NotifyTeams(teamsTemplate, container, reportOrRegression!,
                     notification.NotificationId);
                 break;
             case AdoTemplate adoTemplate when reportOrRegression is not null:
-                return await _context.Ado.NotifyAdo(adoTemplate, container, reportOrRegression, isLastRetryAttempt,
-                    notification.NotificationId);
+                if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableWorkItemCreation)) {
+                    return await _context.Ado.NotifyAdo(adoTemplate, container, reportOrRegression, isLastRetryAttempt,
+                        notification.NotificationId);
+                } else {
+                    return OneFuzzResultVoid.Error(ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED, "Work item processing is currently disabled");
+                }
             case GithubIssuesTemplate githubIssuesTemplate when reportOrRegression is not null:
                 await _context.GithubIssues.GithubIssue(githubIssuesTemplate, container, reportOrRegression,
                     notification.NotificationId);