Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Commit

Permalink
Refactor AdoTemplate rendering (#3370)
Browse files Browse the repository at this point in the history 
* .

* Add tests and complete refactor

* Remove done todo

* Log the query we created
  • Loading branch information
@tevoinea @AdamL-Microsoft
tevoinea authored and AdamL-Microsoft committed Aug 1, 2023
1 parent 638ff8c commit cc182f8
Show file tree
Hide file tree
Showing 4 changed files with 240 additions and 94 deletions.
11 changes: 11 additions & 0 deletions src/ApiService/ApiService/OneFuzzTypes/Model.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,17 @@ public async Task<OneFuzzResultVoid> Validate() {
}
}

public record RenderedAdoTemplate(
Uri BaseUrl,
SecretData<string> AuthToken,
string Project,
string Type,
List<string> UniqueFields,
Dictionary<string, string> AdoFields,
ADODuplicateTemplate OnDuplicate,
string? Comment = null
) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment);

public record TeamsTemplate(SecretData<string> Url) : NotificationTemplate {
public Task<OneFuzzResultVoid> Validate() {
// The only way we can validate in the current state is to send a test webhook
Expand Down
191 changes: 115 additions & 76 deletions src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ public interface IAdo {
}

public class Ado : NotificationsBase, IAdo {
// https://github.com/MicrosoftDocs/azure-devops-docs/issues/5890#issuecomment-539632059
private const int MAX_SYSTEM_TITLE_LENGTH = 128;
private const string TITLE_FIELD = "System.Title";

public Ado(ILogger<Ado> logTracer, IOnefuzzContext context) : base(logTracer, context) {
}

Expand Down Expand Up @@ -52,8 +56,7 @@ public async Async.Task<OneFuzzResultVoid> NotifyAdo(AdoTemplate config, Contain
_logTracer.LogEvent(adoEventType);

try {
var ado = await AdoConnector.AdoConnectorCreator(_context, container, filename, config, report, _logTracer);
await ado.Process(notificationInfo);
await ProcessNotification(_context, container, filename, config, report, _logTracer, notificationInfo);
} catch (Exception e)
when (e is VssUnauthorizedException || e is VssAuthenticationException || e is VssServiceException) {
if (config.AdoFields.TryGetValue("System.AssignedTo", out var assignedTo)) {
Expand Down Expand Up @@ -83,7 +86,7 @@ private static bool IsTransient(Exception e) {
};

var errorStr = e.ToString();
return errorCodes.Any(code => errorStr.Contains(code));
return errorCodes.Any(errorStr.Contains);
}

public static async Async.Task<OneFuzzResultVoid> Validate(AdoTemplate config) {
Expand Down Expand Up @@ -171,56 +174,124 @@ private static async Async.Task<Dictionary<string, WorkItemField2>> GetValidFiel
.ToDictionary(field => field.ReferenceName.ToLowerInvariant());
}

public sealed class AdoConnector {
// https://github.com/MicrosoftDocs/azure-devops-docs/issues/5890#issuecomment-539632059
private const int MAX_SYSTEM_TITLE_LENGTH = 128;
private static async Async.Task ProcessNotification(IOnefuzzContext context, Container container, string filename, AdoTemplate config, Report report, ILogger logTracer, IList<(string, string)> notificationInfo, Renderer? renderer = null) {
if (!config.AdoFields.TryGetValue(TITLE_FIELD, out var issueTitle)) {
issueTitle = "{{ report.crash_site }} - {{ report.executable }}";
}
var instanceUrl = context.Creds.GetInstanceUrl();
renderer ??= await Renderer.ConstructRenderer(context, container, filename, issueTitle, report, instanceUrl, logTracer);
var project = renderer.Render(config.Project, instanceUrl);

private readonly AdoTemplate _config;
private readonly Renderer _renderer;
private readonly string _project;
private readonly WorkItemTrackingHttpClient _client;
private readonly Uri _instanceUrl;
private readonly ILogger _logTracer;
public static async Async.Task<AdoConnector> AdoConnectorCreator(IOnefuzzContext context, Container container, string filename, AdoTemplate config, Report report, ILogger logTracer, Renderer? renderer = null) {
if (!config.AdoFields.TryGetValue("System.Title", out var issueTitle)) {
issueTitle = "{{ report.crash_site }} - {{ report.executable }}";
var authToken = await context.SecretsOperations.GetSecretValue(config.AuthToken.Secret);
var client = GetAdoClient(config.BaseUrl, authToken!);

var renderedConfig = RenderAdoTemplate(logTracer, renderer, config, instanceUrl);
var ado = new AdoConnector(renderedConfig, project!, client, instanceUrl, logTracer, await GetValidFields(client, project));
await ado.Process(notificationInfo);
}

public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer renderer, AdoTemplate original, Uri instanceUrl) {
var adoFields = original.AdoFields.ToDictionary(kvp => kvp.Key, kvp => Render(renderer, kvp.Value, instanceUrl, logTracer));
var onDuplicateAdoFields = original.OnDuplicate.AdoFields.ToDictionary(kvp => kvp.Key, kvp => Render(renderer, kvp.Value, instanceUrl, logTracer));

var systemTitle = renderer.IssueTitle;
if (systemTitle.Length > MAX_SYSTEM_TITLE_LENGTH) {
var systemTitleHashString = Convert.ToHexString(
System.Security.Cryptography.SHA256.HashData(Encoding.UTF8.GetBytes(systemTitle))
);
// try to avoid naming collisions caused by the trim by appending the first 8 characters of the title's hash at the end
var truncatedTitle = $"{systemTitle[..(MAX_SYSTEM_TITLE_LENGTH - 14)]}... [{systemTitleHashString[..8]}]";

// TITLE_FIELD is required in adoFields (ADO won't allow you to create a work item without a title)
adoFields[TITLE_FIELD] = truncatedTitle;

// It may or may not be present in on_duplicate
if (onDuplicateAdoFields.ContainsKey(TITLE_FIELD)) {
onDuplicateAdoFields[TITLE_FIELD] = truncatedTitle;
}
var instanceUrl = context.Creds.GetInstanceUrl();
renderer ??= await Renderer.ConstructRenderer(context, container, filename, issueTitle, report, instanceUrl, logTracer);
var project = renderer.Render(config.Project, instanceUrl);

var authToken = await context.SecretsOperations.GetSecretValue(config.AuthToken.Secret);
var client = GetAdoClient(config.BaseUrl, authToken!);
return new AdoConnector(config, renderer, project!, client, instanceUrl, logTracer);
logTracer.LogInformation(
"System.Title \"{Title}\" was too long ({TitleLength} chars); shortend it to \"{NewTitle}\" ({NewTitleLength} chars)",
systemTitle,
systemTitle.Length,
adoFields[TITLE_FIELD],
adoFields[TITLE_FIELD].Length
);
}

var onDuplicateUnless = original.OnDuplicate.Unless?.Select(dict =>
dict.ToDictionary(kvp => kvp.Key, kvp => Render(renderer, kvp.Value, instanceUrl, logTracer)))
.ToList();

var onDuplicate = new ADODuplicateTemplate(
original.OnDuplicate.Increment,
original.OnDuplicate.SetState,
onDuplicateAdoFields,
original.OnDuplicate.Comment != null ? Render(renderer, original.OnDuplicate.Comment, instanceUrl, logTracer) : null,
onDuplicateUnless
);

return new RenderedAdoTemplate(
original.BaseUrl,
original.AuthToken,
Render(renderer, original.Project, instanceUrl, logTracer),
Render(renderer, original.Type, instanceUrl, logTracer),
original.UniqueFields,
adoFields,
onDuplicate,
original.Comment != null ? Render(renderer, original.Comment, instanceUrl, logTracer) : null
);
}

public AdoConnector(AdoTemplate config, Renderer renderer, string project, WorkItemTrackingHttpClient client, Uri instanceUrl, ILogger logTracer) {
private static string Render(Renderer renderer, string toRender, Uri instanceUrl, ILogger logTracer) {
try {
return renderer.Render(toRender, instanceUrl, strictRendering: true);
} catch {
logTracer.LogWarning("Failed to render template in strict mode. Falling back to relaxed mode. {Template} ", toRender);
return renderer.Render(toRender, instanceUrl, strictRendering: false);
}
}

public sealed class AdoConnector {
private readonly RenderedAdoTemplate _config;
private readonly string _project;
private readonly WorkItemTrackingHttpClient _client;
private readonly ILogger _logTracer;
private readonly Dictionary<string, WorkItemField2> _validFields;

public AdoConnector(RenderedAdoTemplate config, string project, WorkItemTrackingHttpClient client, Uri instanceUrl, ILogger logTracer, Dictionary<string, WorkItemField2> validFields) {
_config = config;
_renderer = renderer;
_project = project;
_client = client;
_instanceUrl = instanceUrl;
_logTracer = logTracer;
_validFields = validFields;
}

public string Render(string template) {
try {
return _renderer.Render(template, _instanceUrl, strictRendering: true);
} catch {
_logTracer.LogWarning("Failed to render template in strict mode. Falling back to relaxed mode. {Template} ", template);
return _renderer.Render(template, _instanceUrl, strictRendering: false);
public async IAsyncEnumerable<WorkItem> ExistingWorkItems(IList<(string, string)> notificationInfo) {
var (wiql, postQueryFilter) = CreateExistingWorkItemsQuery(notificationInfo);
foreach (var workItemReference in (await _client.QueryByWiqlAsync(wiql)).WorkItems) {
var item = await _client.GetWorkItemAsync(_project, workItemReference.Id, expand: WorkItemExpand.All);

var loweredFields = item.Fields.ToDictionary(kvp => kvp.Key.ToLowerInvariant(), kvp => JsonSerializer.Serialize(kvp.Value));
if (postQueryFilter.Any() && !postQueryFilter.All(kvp => {
var lowerKey = kvp.Key.ToLowerInvariant();
return loweredFields.ContainsKey(lowerKey) && loweredFields[lowerKey] == postQueryFilter[kvp.Key];
})) {
continue;
}

yield return item;
}
}

public async IAsyncEnumerable<WorkItem> ExistingWorkItems(IList<(string, string)> notificationInfo) {
public (Wiql, Dictionary<string, string>) CreateExistingWorkItemsQuery(IList<(string, string)> notificationInfo) {
var filters = new Dictionary<string, string>();
foreach (var key in _config.UniqueFields) {
var filter = string.Empty;
if (string.Equals("System.TeamProject", key)) {
filter = Render(_config.Project);
filter = _config.Project;
} else if (_config.AdoFields.TryGetValue(key, out var field)) {
filter = Render(field);
filter = field;
} else {
_logTracer.AddTags(notificationInfo);
_logTracer.LogError("Failed to check for existing work items using the UniqueField Key: {Key}. Value is not present in config field AdoFields.", key);
Expand All @@ -230,9 +301,6 @@ public async IAsyncEnumerable<WorkItem> ExistingWorkItems(IList<(string, string)
filters.Add(key.ToLowerInvariant(), filter);
}

var project = filters.TryGetValue("system.teamproject", out var value) ? value : null;
var validFields = await GetValidFields(_client, project);

var postQueryFilter = new Dictionary<string, string>();
/*
# WIQL (Work Item Query Language) is an SQL like query language that
Expand All @@ -246,12 +314,12 @@ public async IAsyncEnumerable<WorkItem> ExistingWorkItems(IList<(string, string)
var parts = new List<string>();
foreach (var key in filters.Keys) {
//# Only add pre-system approved fields to the query
if (!validFields.ContainsKey(key)) {
if (!_validFields.ContainsKey(key)) {
postQueryFilter.Add(key, filters[key]);
continue;
}

var field = validFields[key];
var field = _validFields[key];
var operation = GetSupportedOperation(field);
if (operation.IsOk) {
/*
Expand All @@ -278,23 +346,10 @@ public async IAsyncEnumerable<WorkItem> ExistingWorkItems(IList<(string, string)
query += " where " + string.Join(" AND ", parts);
}

var wiql = new Wiql() {
_logTracer.LogInformation("{Query}", query);
return (new Wiql() {
Query = query
};

foreach (var workItemReference in (await _client.QueryByWiqlAsync(wiql)).WorkItems) {
var item = await _client.GetWorkItemAsync(_project, workItemReference.Id, expand: WorkItemExpand.All);

var loweredFields = item.Fields.ToDictionary(kvp => kvp.Key.ToLowerInvariant(), kvp => JsonSerializer.Serialize(kvp.Value));
if (postQueryFilter.Any() && !postQueryFilter.All(kvp => {
var lowerKey = kvp.Key.ToLowerInvariant();
return loweredFields.ContainsKey(lowerKey) && loweredFields[lowerKey] == postQueryFilter[kvp.Key];
})) {
continue;
}

yield return item;
}
}, postQueryFilter);
}

/// <returns>true if the state of the item was modified</returns>
Expand All @@ -308,7 +363,7 @@ public async Async.Task<bool> UpdateExisting(WorkItem item, IList<(string, strin
}

if (_config.OnDuplicate.Comment != null) {
var comment = Render(_config.OnDuplicate.Comment);
var comment = _config.OnDuplicate.Comment;
_ = await _client.AddCommentAsync(
new CommentCreate() {
Text = comment
Expand All @@ -329,7 +384,7 @@ public async Async.Task<bool> UpdateExisting(WorkItem item, IList<(string, strin
}

foreach (var field in _config.OnDuplicate.AdoFields) {
var fieldValue = Render(_config.OnDuplicate.AdoFields[field.Key]);
var fieldValue = _config.OnDuplicate.AdoFields[field.Key];
document.Add(new JsonPatchOperation() {
Operation = VisualStudio.Services.WebApi.Patch.Operation.Replace,
Path = $"/fields/{field.Key}",
Expand Down Expand Up @@ -372,14 +427,14 @@ private bool MatchesUnlessCase(WorkItem workItem) =>
// All fields within the condition must match
.All(kvp =>
workItem.Fields.TryGetValue<string>(kvp.Key, out var value) &&
string.Equals(Render(kvp.Value), value, StringComparison.OrdinalIgnoreCase)));
string.Equals(kvp.Value, value, StringComparison.OrdinalIgnoreCase)));

private async Async.Task<WorkItem> CreateNew() {
var (taskType, document) = RenderNew();
var entry = await _client.CreateWorkItemAsync(document, _project, taskType);

if (_config.Comment != null) {
var comment = Render(_config.Comment);
var comment = _config.Comment;
_ = await _client.AddCommentAsync(
new CommentCreate() {
Text = comment,
Expand All @@ -391,7 +446,7 @@ private async Async.Task<WorkItem> CreateNew() {
}

private (string, JsonPatchDocument) RenderNew() {
var taskType = Render(_config.Type);
var taskType = _config.Type;
var document = new JsonPatchDocument();
if (!_config.AdoFields.ContainsKey("System.Tags")) {
document.Add(new JsonPatchOperation() {
Expand All @@ -401,24 +456,8 @@ private async Async.Task<WorkItem> CreateNew() {
});
}

var systemTitle = _renderer.IssueTitle;
if (systemTitle.Length > MAX_SYSTEM_TITLE_LENGTH) {
var systemTitleHashString = Convert.ToHexString(
System.Security.Cryptography.SHA256.HashData(Encoding.UTF8.GetBytes(systemTitle))
);
// try to avoid naming collisions caused by the trim by appending the first 8 characters of the title's hash at the end
_config.AdoFields["System.Title"] = $"{systemTitle[..(MAX_SYSTEM_TITLE_LENGTH - 14)]}... [{systemTitleHashString[..8]}]";
_logTracer.LogInformation(
"System.Title \"{Title}\" was too long ({TitleLength} chars); shortend it to \"{NewTitle}\" ({NewTitleLength} chars)",
systemTitle,
systemTitle.Length,
_config.AdoFields["System.Title"],
_config.AdoFields["System.Title"].Length
);
}

foreach (var field in _config.AdoFields.Keys) {
var value = Render(_config.AdoFields[field]);
var value = _config.AdoFields[field];

if (string.Equals(field, "System.Tags")) {
value += ";Onefuzz";
Expand Down
Loading

0 comments on commit cc182f8

Please sign in to comment.