Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Commit

Permalink
Merge remote-tracking branch 'microsoft/main' into coverage-example-s…
Browse files Browse the repository at this point in the history 
…ourceline
  • Loading branch information
@ranweiler
ranweiler committed Jan 20, 2023
2 parents cd905e0 + 62a0dd6 commit d470ced
Show file tree
Hide file tree
Showing 26 changed files with 431 additions and 224 deletions.
1 change: 1 addition & 0 deletions src/ApiService/ApiService/Functions/ValidateScriban.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymou
null,
null,
null,
null,
null
);

Expand Down
18 changes: 14 additions & 4 deletions src/ApiService/ApiService/OneFuzzTypes/Events.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,17 +302,27 @@ public record EventCrashReported(
Container Container,
[property: JsonPropertyName("filename")] String FileName,
TaskConfig? TaskConfig
) : BaseEvent();

) : BaseEvent(), ITruncatable<BaseEvent> {
public BaseEvent Truncate(int maxLength) {
return this with {
Report = Report.Truncate(maxLength)
};
}
}

[EventType(EventType.RegressionReported)]
public record EventRegressionReported(
RegressionReport RegressionReport,
Container Container,
[property: JsonPropertyName("filename")] String FileName,
TaskConfig? TaskConfig
) : BaseEvent();

) : BaseEvent(), ITruncatable<BaseEvent> {
public BaseEvent Truncate(int maxLength) {
return this with {
RegressionReport = RegressionReport.Truncate(maxLength)
};
}
}

[EventType(EventType.FileAdded)]
public record EventFileAdded(
Expand Down
63 changes: 56 additions & 7 deletions src/ApiService/ApiService/OneFuzzTypes/Model.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,8 +457,31 @@ public record Report(
string? MinimizedStackFunctionLinesSha256,
string? ToolName,
string? ToolVersion,
string? OnefuzzVersion
) : IReport;
string? OnefuzzVersion,
Uri? ReportUrl
) : IReport, ITruncatable<Report> {
public Report Truncate(int maxLength) {
return this with {
Executable = Executable[..maxLength],
CrashType = CrashType[..Math.Min(maxLength, CrashType.Length)],
CrashSite = CrashSite[..Math.Min(maxLength, CrashSite.Length)],
CallStack = TruncateUtils.TruncateList(CallStack, maxLength),
CallStackSha256 = CallStackSha256[..Math.Min(maxLength, CallStackSha256.Length)],
InputSha256 = InputSha256[..Math.Min(maxLength, InputSha256.Length)],
AsanLog = AsanLog?[..Math.Min(maxLength, AsanLog.Length)],
ScarinessDescription = ScarinessDescription?[..Math.Min(maxLength, ScarinessDescription.Length)],
MinimizedStack = MinimizedStack != null ? TruncateUtils.TruncateList(MinimizedStack, maxLength) : MinimizedStack,
MinimizedStackSha256 = MinimizedStackSha256?[..Math.Min(maxLength, MinimizedStackSha256.Length)],
MinimizedStackFunctionNames = MinimizedStackFunctionNames != null ? TruncateUtils.TruncateList(MinimizedStackFunctionNames, maxLength) : MinimizedStackFunctionNames,
MinimizedStackFunctionNamesSha256 = MinimizedStackFunctionNamesSha256?[..Math.Min(maxLength, MinimizedStackFunctionNamesSha256.Length)],
MinimizedStackFunctionLines = MinimizedStackFunctionLines != null ? TruncateUtils.TruncateList(MinimizedStackFunctionLines, maxLength) : MinimizedStackFunctionLines,
MinimizedStackFunctionLinesSha256 = MinimizedStackFunctionLinesSha256?[..Math.Min(maxLength, MinimizedStackFunctionLinesSha256.Length)],
ToolName = ToolName?[..Math.Min(maxLength, ToolName.Length)],
ToolVersion = ToolVersion?[..Math.Min(maxLength, ToolVersion.Length)],
OnefuzzVersion = OnefuzzVersion?[..Math.Min(maxLength, OnefuzzVersion.Length)],
};
}
}

public record NoReproReport(
string InputSha,
Expand All @@ -468,18 +491,40 @@ public record NoReproReport(
Guid JobId,
long Tries,
string? Error
);
) : ITruncatable<NoReproReport> {
public NoReproReport Truncate(int maxLength) {
return this with {
Executable = Executable?[..maxLength],
Error = Error?[..maxLength]
};
}
}

public record CrashTestResult(
Report? CrashReport,
NoReproReport? NoReproReport
);
) : ITruncatable<CrashTestResult> {
public CrashTestResult Truncate(int maxLength) {
return new CrashTestResult(
CrashReport?.Truncate(maxLength),
NoReproReport?.Truncate(maxLength)
);
}
}

public record RegressionReport(
CrashTestResult CrashTestResult,
CrashTestResult? OriginalCrashTestResult
) : IReport;

CrashTestResult? OriginalCrashTestResult,
Uri? ReportUrl
) : IReport, ITruncatable<RegressionReport> {
public RegressionReport Truncate(int maxLength) {
return new RegressionReport(
CrashTestResult.Truncate(maxLength),
OriginalCrashTestResult?.Truncate(maxLength),
ReportUrl
);
}
}

[JsonConverter(typeof(NotificationTemplateConverter))]
#pragma warning disable CA1715
Expand Down Expand Up @@ -968,3 +1013,7 @@ public record TemplateRenderContext(
string ReportFilename,
string ReproCmd
);

public interface ITruncatable<T> {
public T Truncate(int maxLength);
}
41 changes: 19 additions & 22 deletions src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,34 +20,31 @@ public NotificationOperations(ILogTracer log, IOnefuzzContext context)
public async Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt) {
var notifications = GetNotifications(container);
var hasNotifications = await notifications.AnyAsync();

if (!hasNotifications) {
return;
}

var reportOrRegression = await _context.Reports.GetReportOrRegression(container, filename, expectReports: hasNotifications);
var done = new List<NotificationTemplate>();
await foreach (var notification in notifications) {
if (done.Contains(notification.Config)) {
continue;
}
if (hasNotifications) {
var done = new List<NotificationTemplate>();
await foreach (var notification in notifications) {
if (done.Contains(notification.Config)) {
continue;
}

done.Add(notification.Config);
done.Add(notification.Config);

if (notification.Config is TeamsTemplate teamsTemplate) {
await _context.Teams.NotifyTeams(teamsTemplate, container, filename, reportOrRegression!, notification.NotificationId);
}
if (notification.Config is TeamsTemplate teamsTemplate) {
await _context.Teams.NotifyTeams(teamsTemplate, container, filename, reportOrRegression!, notification.NotificationId);
}

if (reportOrRegression == null) {
continue;
}
if (reportOrRegression == null) {
continue;
}

if (notification.Config is AdoTemplate adoTemplate) {
await _context.Ado.NotifyAdo(adoTemplate, container, filename, reportOrRegression, isLastRetryAttempt, notification.NotificationId);
}
if (notification.Config is AdoTemplate adoTemplate) {
await _context.Ado.NotifyAdo(adoTemplate, container, filename, reportOrRegression, isLastRetryAttempt, notification.NotificationId);
}

if (notification.Config is GithubIssuesTemplate githubIssuesTemplate) {
await _context.GithubIssues.GithubIssue(githubIssuesTemplate, container, filename, reportOrRegression, notification.NotificationId);
if (notification.Config is GithubIssuesTemplate githubIssuesTemplate) {
await _context.GithubIssues.GithubIssue(githubIssuesTemplate, container, filename, reportOrRegression, notification.NotificationId);
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public ProxyOperations(ILogTracer log, IOnefuzzContext context)

public async Async.Task<Proxy> GetOrCreate(Region region) {
{
var proxyList = QueryAsync(filter: TableClient.CreateQueryFilter($"region eq {region.String} and outdated eq false"));
var proxyList = QueryAsync(filter: TableClient.CreateQueryFilter($"RowKey eq {region.String} and outdated eq false"));
await foreach (var proxy in proxyList) {
if (IsOutdated(proxy)) {
var r1 = await Replace(proxy with { Outdated = true });
Expand Down
16 changes: 11 additions & 5 deletions src/ApiService/ApiService/onefuzzlib/Reports.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,21 +44,27 @@ public Reports(ILogTracer log, IContainers containers) {
return null;
}

return ParseReportOrRegression(blob.ToString(), filePath, expectReports);
var reportUrl = await _containers.GetFileUrl(container, fileName, StorageType.Corpus);

return ParseReportOrRegression(blob.ToString(), filePath, reportUrl, expectReports);
}

private IReport? ParseReportOrRegression(string content, string? filePath, bool expectReports = false) {
private IReport? ParseReportOrRegression(string content, string? filePath, Uri? reportUrl, bool expectReports = false) {
var regressionReport = JsonSerializer.Deserialize<RegressionReport>(content, EntityConverter.GetJsonSerializerOptions());
if (regressionReport == null || regressionReport.CrashTestResult == null) {
var report = JsonSerializer.Deserialize<Report>(content, EntityConverter.GetJsonSerializerOptions());
if (expectReports && report == null) {
_log.Error($"unable to parse report ({filePath:Tag:FilePath}) as a report or regression");
return null;
}
return report;
return report != null ? report with { ReportUrl = reportUrl } : report;
}
return regressionReport;
return regressionReport != null ? regressionReport with { ReportUrl = reportUrl } : regressionReport;
}
}

public interface IReport { };
public interface IReport {
Uri? ReportUrl {
init;
}
};
7 changes: 7 additions & 0 deletions src/ApiService/ApiService/onefuzzlib/Utils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,10 @@ public static async IAsyncEnumerable<List<TSource>> Chunk<TSource>(this IAsyncEn
}
}
}

public static class TruncateUtils {
public static List<string> TruncateList(List<string> data, int maxLength) {
int currentLength = 0;
return data.TakeWhile(curr => (currentLength += curr.Length) <= maxLength).ToList();
}
}
16 changes: 14 additions & 2 deletions src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ async public Async.Task SendEvent(EventMessage eventMessage) {
}

async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) {
(string, string)[] tags = { ("WebhookId", webhook.WebhookId.ToString()), ("EventId", eventMessage.EventId.ToString()) };

var message = new WebhookMessageLog(
EventId: eventMessage.EventId,
EventType: eventMessage.EventType,
Expand All @@ -46,8 +48,18 @@ async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) {

var r = await _context.WebhookMessageLogOperations.Replace(message);
if (!r.IsOk) {
_logTracer.WithHttpStatus(r.ErrorV).Error($"Failed to replace webhook message log {webhook.WebhookId:Tag:WebhookId} - {eventMessage.EventId:Tag:EventId}");
if (r.ErrorV.Reason.Contains("The entity is larger than the maximum allowed size") && eventMessage.Event is ITruncatable<BaseEvent> truncatableEvent) {
_logTracer.WithTags(tags).Warning($"The WebhookMessageLog was too long. Truncating event data and trying again.");
var truncatedEventMessage = message with {
Event = truncatableEvent.Truncate(1000)
};
r = await _context.WebhookMessageLogOperations.Replace(truncatedEventMessage);
}
if (!r.IsOk) {
_logTracer.WithHttpStatus(r.ErrorV).WithTags(tags).Error($"Failed to replace webhook message log {webhook.WebhookId:Tag:WebhookId} - {eventMessage.EventId:Tag:EventId}");
}
}

await _context.WebhookMessageLogOperations.QueueWebhook(message);
}

Expand All @@ -57,7 +69,7 @@ public async Async.Task<bool> Send(WebhookMessageLog messageLog) {
throw new Exception($"Invalid Webhook. Webhook with WebhookId: {messageLog.WebhookId} Not Found");
}

var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat);
var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event!, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat);

var headers = new Dictionary<string, string> { { "User-Agent", $"onefuzz-webhook {_context.ServiceConfiguration.OneFuzzVersion}" } };

Expand Down
Loading

0 comments on commit d470ced

Please sign in to comment.