From 5c0f0ab6f33a67834ec00bd8513d86e9a3f27bbf Mon Sep 17 00:00:00 2001 From: Kanan B <32438208+kananb@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:40:42 -0700 Subject: [PATCH] Support custom ado fields that mark work items as duplicate (#3467) * Add field to ado config for checking duplicate work items * Make duplicate fields nullable and add it to python models * Update broken tests * Update docs to include new ado_duplicate_fields property --- .../ado-work-items.json | 4 ++++ docs/notifications/ado.md | 7 +++++++ src/ApiService/ApiService/OneFuzzTypes/Model.cs | 4 +++- .../ApiService/onefuzzlib/notifications/Ado.cs | 13 +++++++++---- .../JinjaToScribanMigrationTests.cs | 2 ++ src/ApiService/Tests/OrmModelsTest.cs | 2 ++ src/pytypes/onefuzztypes/models.py | 1 + 7 files changed, 28 insertions(+), 5 deletions(-) diff --git a/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json b/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json index eb89fc019d..034d97cf15 100644 --- a/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json +++ b/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json @@ -13,6 +13,10 @@ "System.AreaPath": "OneFuzz-Ado-Integration", "System.Title": "{{report.task_id}}" }, + "ado_duplicate_fields": { + "System.Reason": "My custom value that means a work item is a duplicate", + "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate" + }, "on_duplicate": { "increment": [], "comment": "DUP {{report.input_sha256}}
Repro Command: 
 {{ repro_cmd }}
", diff --git a/docs/notifications/ado.md b/docs/notifications/ado.md index 131986afba..09dd5b9072 100644 --- a/docs/notifications/ado.md +++ b/docs/notifications/ado.md @@ -51,6 +51,13 @@ clickable, make it a link. "System.Title": "{{ report.crash_site }} - {{ report.executable }}", "Microsoft.VSTS.TCM.ReproSteps": "This is my call stack: " }, + "ado_duplicate_fields": { + "System.Reason": "My custom value that means a work item is a duplicate", + "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate" + // note: the fields and values below are checked by default and don't need to be specified + // "System.Reason": "Duplicate" + // "Microsoft.VSTS.Common.ResolvedReason": "Duplicate" + }, "comment": "This is my comment. {{ report.input_sha256 }} {{ input_url }} 
{{ repro_cmd }}
", "unique_fields": ["System.Title", "System.AreaPath"], "on_duplicate": { diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index b839f52ddc..424669899a 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -689,6 +689,7 @@ public record AdoTemplate( List UniqueFields, Dictionary AdoFields, ADODuplicateTemplate OnDuplicate, + Dictionary? AdoDuplicateFields = null, string? Comment = null ) : NotificationTemplate { public async Task Validate() { @@ -704,8 +705,9 @@ public record RenderedAdoTemplate( List UniqueFields, Dictionary AdoFields, ADODuplicateTemplate OnDuplicate, + Dictionary? AdoDuplicateFields = null, string? Comment = null - ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment); + ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, AdoDuplicateFields, Comment); public record TeamsTemplate(SecretData Url) : NotificationTemplate { public Task Validate() { diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs index e05bb9bc24..98b857c9bc 100644 --- a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs +++ b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs @@ -239,7 +239,7 @@ private static async Async.Task ProcessNotification(IOnefuzzContext context, Con var renderedConfig = RenderAdoTemplate(logTracer, renderer, config, instanceUrl); var ado = new AdoConnector(renderedConfig, project!, client, instanceUrl, logTracer, await GetValidFields(client, project)); - await ado.Process(notificationInfo); + await ado.Process(notificationInfo, config.AdoDuplicateFields); } public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer renderer, AdoTemplate original, Uri instanceUrl) { @@ -291,6 +291,7 @@ public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer original.UniqueFields, adoFields, onDuplicate, + original.AdoDuplicateFields, original.Comment != null ? Render(renderer, original.Comment, instanceUrl, logTracer) : null ); } @@ -525,7 +526,7 @@ private async Async.Task CreateNew() { return (taskType, document); } - public async Async.Task Process(IList<(string, string)> notificationInfo) { + public async Async.Task Process(IList<(string, string)> notificationInfo, Dictionary? duplicateFields) { var updated = false; WorkItem? oldestWorkItem = null; await foreach (var workItem in ExistingWorkItems(notificationInfo)) { @@ -535,7 +536,7 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) { _logTracer.AddTags(new List<(string, string)> { ("MatchingWorkItemIds", $"{workItem.Id}") }); _logTracer.LogInformation("Found matching work item"); } - if (IsADODuplicateWorkItem(workItem)) { + if (IsADODuplicateWorkItem(workItem, duplicateFields)) { continue; } @@ -575,13 +576,17 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) { } } - private static bool IsADODuplicateWorkItem(WorkItem wi) { + private static bool IsADODuplicateWorkItem(WorkItem wi, Dictionary? duplicateFields) { // A work item could have System.State == Resolve && System.Reason == Duplicate // OR it could have System.State == Closed && System.Reason == Duplicate // I haven't found any other combinations where System.Reason could be duplicate but just to be safe // we're explicitly _not_ checking the state of the work item to determine if it's duplicate return wi.Fields.ContainsKey("System.Reason") && string.Equals(wi.Fields["System.Reason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase) || wi.Fields.ContainsKey("Microsoft.VSTS.Common.ResolvedReason") && string.Equals(wi.Fields["Microsoft.VSTS.Common.ResolvedReason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase) + || duplicateFields?.Any(fieldPair => { + var (field, value) = fieldPair; + return wi.Fields.ContainsKey(field) && string.Equals(wi.Fields[field].ToString(), value, StringComparison.OrdinalIgnoreCase); + }) == true // Alternatively, the work item can also specify a 'relation' to another work item. // This is typically used to create parent/child relationships between work items but can also // Be used to mark duplicates so we should check this as well. diff --git a/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs b/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs index 0ae3b11cb5..4033a05369 100644 --- a/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs +++ b/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs @@ -111,6 +111,7 @@ public async Async.Task OptionalFieldsAreSupported() { }, "{{ if org }} blah {{ end }}" ), + null, "{{ if org }} blah {{ end }}" ); @@ -137,6 +138,7 @@ public async Async.Task All_ADO_Fields_Are_Migrated() { }, "{% if org %} comment {% endif %}" ), + null, "{% if org %} comment {% endif %}" ); diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index 1aa7d2d163..956d0c30c5 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -232,6 +232,7 @@ from authToken in Arb.Generate>() from str in Arb.Generate() from fields in Arb.Generate>() from adoFields in Arb.Generate>() + from adoDuplicateFields in Arb.Generate>() from dupeTemplate in Arb.Generate() select new AdoTemplate( baseUrl, @@ -241,6 +242,7 @@ from dupeTemplate in Arb.Generate() fields, adoFields, dupeTemplate, + adoDuplicateFields, str.Get)); public static Arbitrary ArbTeamsTemplate() diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index a5f8139e97..c888621600 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -273,6 +273,7 @@ class ADOTemplate(BaseModel): unique_fields: List[str] comment: Optional[str] ado_fields: Dict[str, str] + ado_duplicate_fields: Optional[Dict[str, str]] on_duplicate: ADODuplicateTemplate # validator needed to convert auth_token to SecretData