Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Commit

Permalink
Merge branch 'main' into report-all-loader-errors
Browse files Browse the repository at this point in the history
  • Loading branch information
Porges authored Oct 11, 2023
2 parents e776555 + 2f2a649 commit 3322f45
Show file tree
Hide file tree
Showing 10 changed files with 375 additions and 32 deletions.
29 changes: 29 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,35 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## 8.9.0

### Added

* Agent: Added fuzz tests for coverage recording [#3322](https://github.com/microsoft/onefuzz/pull/3322)
* Agent: Added version checking in local tasks [#3517](https://github.com/microsoft/onefuzz/pull/3517)
* Agent: Create directories from template specification in local task if they don't exist [#3522](https://github.com/microsoft/onefuzz/pull/3522)
* CLI: Added a new command for template creation in the local task `onefuzz-task local create-template` [#3531](https://github.com/microsoft/onefuzz/pull/3531)
* CLI/Deployment/Service: Support for retention policies on containers [#3501](https://github.com/microsoft/onefuzz/pull/3501)
* Service: Add onefuzz service version to job created events [#3504](https://github.com/microsoft/onefuzz/pull/3504)
* Service: Added a start time to job and task records [#3440](https://github.com/microsoft/onefuzz/pull/3440)

### Changed

* Agent: Improved handling of unexpected breakpoints [#3493](https://github.com/microsoft/onefuzz/pull/3493)
* Agent: Updated windows interceptor list [#3528](https://github.com/microsoft/onefuzz/pull/3528), [#3549](https://github.com/microsoft/onefuzz/pull/3549)
* Agent: Reporting coverage on task start up, ensuring `coverage_data` is emitted at the beginning of every task instead of when `new_coverage` is identified [#3502](https://github.com/microsoft/onefuzz/pull/3502)
* CLI/Deployment: Updating onefuzz cli requirements.txt to accept `>= onefuzztypes` versions [#3477](https://github.com/microsoft/onefuzz/pull/3477), [#3486](https://github.com/microsoft/onefuzz/pull/3486)
* Service: Improve area/iteration path validation in notifications [#3489](https://github.com/microsoft/onefuzz/pull/3489)
* Service: Remove feature flag from heartbeat metrics [#3505](https://github.com/microsoft/onefuzz/pull/3505)

### Fixed

* Agent: Terminate process on timeout in Windows agents for the coverage task [#3529](https://github.com/microsoft/onefuzz/pull/3529)
* Agent/Service: Bumped several C#, Python, and Rust dependencies [#3425](https://github.com/microsoft/onefuzz/pull/3425), [#3424](https://github.com/microsoft/onefuzz/pull/3424), [#3411](https://github.com/microsoft/onefuzz/pull/3411), [#3437](https://github.com/microsoft/onefuzz/pull/3437), [#3436](https://github.com/microsoft/onefuzz/pull/3436), [#3435](https://github.com/microsoft/onefuzz/pull/3435), [#3478](https://github.com/microsoft/onefuzz/pull/3478), [#3484](https://github.com/microsoft/onefuzz/pull/3484), [#3414](https://github.com/microsoft/onefuzz/pull/3414), [#3474](https://github.com/microsoft/onefuzz/pull/3474), [#3434](https://github.com/microsoft/onefuzz/pull/3434), [#3488](https://github.com/microsoft/onefuzz/pull/3488), [#3503](https://github.com/microsoft/onefuzz/pull/3503), [#3520](https://github.com/microsoft/onefuzz/pull/3520), [#3521](https://github.com/microsoft/onefuzz/pull/3521)
* Agent: Fixed dependencies in `onefuzz-task` [#3552](https://github.com/microsoft/onefuzz/pull/3552)
* Service: Removed unnecessary method argument in notifications processing [#3473](https://github.com/microsoft/onefuzz/pull/3473)
* Service: Ignore regression work item updates when the work item is in some states [#3532](https://github.com/microsoft/onefuzz/pull/3532)

## 8.8.0

### Added
Expand Down
2 changes: 1 addition & 1 deletion CURRENT_VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
8.8.0
8.9.0
1 change: 1 addition & 0 deletions src/ApiService/ApiService/FeatureFlags.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ public static class FeatureFlagConstants {
public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention";
public const string EnableWorkItemCreation = "EnableWorkItemCreation";
public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies";
public const string EnableSlimEventSerialization = "EnableSlimEventSerialization";
}
25 changes: 12 additions & 13 deletions src/ApiService/ApiService/onefuzzlib/Events.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ public class Events : IEvents {
private readonly IContainers _containers;
private readonly ICreds _creds;
private readonly JsonSerializerOptions _options;
private readonly JsonSerializerOptions _optionsSlim;
private readonly JsonSerializerOptions _deserializingFromBlobOptions;
private readonly IOnefuzzContext _context;

public Events(ILogger<Events> log, IOnefuzzContext context) {
_queue = context.Queue;
Expand All @@ -47,9 +49,12 @@ public Events(ILogger<Events> log, IOnefuzzContext context) {
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
_options.Converters.Add(new RemoveUserInfo());
_optionsSlim = new JsonSerializerOptions(_options);
_optionsSlim.Converters.Add(new EventExportConverter<DownloadableEventMessage>());
_deserializingFromBlobOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()) {
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
_context = context;
}

public virtual async Async.Task QueueSignalrEvent(DownloadableEventMessage message) {
Expand All @@ -58,7 +63,13 @@ public virtual async Async.Task QueueSignalrEvent(DownloadableEventMessage messa
("event_id", message.EventId.ToString())
};
var ev = new SignalREvent("events", new List<DownloadableEventMessage>() { message });
var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: _options);

var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch {
true => _optionsSlim,
false => _options
};

var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: opts);

if (!queueResult) {
_log.AddTags(tags);
Expand Down Expand Up @@ -155,16 +166,4 @@ await _containers.SaveBlob(
);
}
}


public class RemoveUserInfo : JsonConverter<UserInfo> {
public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
throw new NotSupportedException("reading UserInfo is not supported");
}

public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) {
writer.WriteStartObject();
writer.WriteEndObject();
}
}
}
13 changes: 11 additions & 2 deletions src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,15 @@ public interface IWebhookOperations : IOrm<Webhook> {
public class WebhookOperations : Orm<Webhook>, IWebhookOperations {

private readonly IHttpClientFactory _httpFactory;
private readonly JsonSerializerOptions _options;
private readonly JsonSerializerOptions _optionsSlim;

public WebhookOperations(IHttpClientFactory httpFactory, ILogger<WebhookOperations> log, IOnefuzzContext context)
: base(log, context) {
_httpFactory = httpFactory;
_options = EntityConverter.GetJsonSerializerOptions();
_optionsSlim = new JsonSerializerOptions(_options);
_optionsSlim.Converters.Add(new EventExportConverter<WebhookMessage>());
}

public async Async.Task SendEvent(DownloadableEventMessage eventMessage) {
Expand Down Expand Up @@ -139,11 +144,15 @@ public async Task<EventPing> Ping(Webhook webhook) {
string data;
var instanceId = await _context.Containers.GetInstanceId();
var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName(), CreatedAt: eventData.CreatedAt, SasUrl: eventData.SasUrl);
var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch {
true => _optionsSlim,
false => _options
};
if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) {
var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookMessage, DataVersion: "2.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) };
data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions());
data = JsonSerializer.Serialize(eventGridMessage, options: opts);
} else {
data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions());
data = JsonSerializer.Serialize(webhookMessage, options: opts);
}

string? digest = null;
Expand Down
76 changes: 76 additions & 0 deletions src/ApiService/ApiService/onefuzzlib/events/Converters.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
using System.Collections;
using System.Reflection;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace Microsoft.OneFuzz.Service {
public class RemoveUserInfo : JsonConverter<UserInfo> {
public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
throw new NotSupportedException("reading UserInfo is not supported");
}

public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) {
writer.WriteStartObject();
writer.WriteEndObject();
}
}

/// <summary>
/// <b>THIS IS A WRITE ONLY JSON CONVERTER</b>
/// <br/>
/// It should only be used when serializing events to be sent outside of the service
/// </summary>
public class EventExportConverter<T> : JsonConverter<T>
where T : DownloadableEventMessage {
public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
throw new NotSupportedException("This converter should only be used when serializing event messages to sent outside of the service");
}

public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) {
BoundedSerializer.WriteInternal(writer, value, options);
}
}

public class BoundedSerializer {
private static HashSet<Type> boundedTypes = new HashSet<Type>{
typeof(Guid),
typeof(DateTime),
typeof(int),
typeof(bool),
typeof(float),
typeof(double),
typeof(long),
typeof(char),
typeof(Uri)
};

public static void WriteInternal(Utf8JsonWriter writer, object type, JsonSerializerOptions options) {
writer.WriteStartObject();
var properties = type.GetType().GetProperties();
foreach (var property in properties) {
if (property.GetValue(type, null) == null
|| typeof(IEnumerable).IsAssignableFrom(property.PropertyType)
|| type.GetType() == property.PropertyType) {
continue;
}
if (HasBoundedSerialization(property)) {
var serialized = JsonSerializer.Serialize(property.GetValue(type, null), property.PropertyType, options);
if (!string.IsNullOrEmpty(serialized)) {
writer.WritePropertyName(property.Name);
writer.WriteRawValue(serialized);
}
} else if (property.PropertyType.IsClass) {
writer.WritePropertyName(property.Name);
WriteInternal(writer, property.GetValue(type, null)!, options);
}
}
writer.WriteEndObject();
}

public static bool HasBoundedSerialization(PropertyInfo propertyInfo) {
return propertyInfo.PropertyType.IsEnum ||
boundedTypes.Contains(propertyInfo.PropertyType) ||
typeof(IValidatedString).IsAssignableFrom(propertyInfo.PropertyType);
}
}
}
Loading

0 comments on commit 3322f45

Please sign in to comment.