Skip to content

Commit

Permalink
new github reporter (#1885)
Browse files Browse the repository at this point in the history
* new github reporter
  • Loading branch information
rogeralsing authored Dec 5, 2022
1 parent bb4e3b1 commit 2107017
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 157 deletions.
31 changes: 15 additions & 16 deletions tests/Proto.Cluster.Tests/ClusterFixture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using ClusterTest.Messages;
Expand Down Expand Up @@ -38,6 +39,8 @@ public interface IClusterFixture
public Task<Cluster> SpawnNode();

Task RemoveNode(Cluster member, bool graceful = true);

Task Trace(Func<Task> test, [CallerMemberName] string testName = "");
}

public static class TracingSettings
Expand All @@ -57,6 +60,7 @@ public abstract class ClusterFixture : IAsyncLifetime, IClusterFixture, IAsyncDi
private readonly ILogger _logger = Log.CreateLogger(nameof(GetType));
private readonly List<Cluster> _members = new();
private static TracerProvider? _tracerProvider;
private GithubActionsReporter _reporter;

protected readonly string ClusterName;

Expand All @@ -65,13 +69,14 @@ static ClusterFixture()
TracingSettings.OpenTelemetryUrl = Environment.GetEnvironmentVariable("OPENTELEMETRY_URL");
TracingSettings.TraceViewUrl = Environment.GetEnvironmentVariable("TRACEVIEW_URL");
TracingSettings.EnableTracing = TracingSettings.OpenTelemetryUrl != null;

//TODO: check if this helps low resource envs like github actions.
ThreadPool.SetMinThreads(40, 40);
}

protected ClusterFixture(int clusterSize, Func<ClusterConfig, ClusterConfig>? configure = null)
protected ClusterFixture( int clusterSize, Func<ClusterConfig, ClusterConfig>? configure = null)
{
_reporter = new GithubActionsReporter(GetType().Name);
#if NETCOREAPP3_1
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
#endif
Expand Down Expand Up @@ -117,22 +122,11 @@ public async Task DisposeAsync()
{
try
{
await _reporter.WriteReportFile();

await OnDisposing();

if (TracingSettings.EnableTracing)
{
var testName = this.GetType().Name;
using (Tracing.StartActivity("ClusterFixture.DisposeAsync " + testName))
{
await WaitForMembersToShutdown();
}
_tracerProvider?.ForceFlush(1000);
}
else
{
await WaitForMembersToShutdown();
}
await WaitForMembersToShutdown();

Members.Clear(); // prevent multiple shutdown attempts if dispose is called multiple times
}
Expand Down Expand Up @@ -189,6 +183,11 @@ public async Task RemoveNode(Cluster member, bool graceful = true)
}
}

public Task Trace(Func<Task> test, string testName = "")
{
return _reporter.Run(test, testName);
}

/// <summary>
/// Spawns a node, adds it to the cluster and member list
/// </summary>
Expand Down Expand Up @@ -243,7 +242,7 @@ private static void InitOpenTelemetryTracing()
.AddService("Proto.Cluster.Tests")
)
.AddProtoActorInstrumentation()
.AddSource(Tracing.ActivitySourceName)
.AddSource(GithubActionsReporter.ActivitySourceName)
.AddOtlpExporter(options =>
{
options.Endpoint = endpoint;
Expand Down
8 changes: 8 additions & 0 deletions tests/Proto.Cluster.Tests/ClusterTestBase.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Proto.Logging;
using Xunit;
using Xunit.Abstractions;

namespace Proto.Cluster.Tests;

Expand All @@ -18,6 +21,11 @@ protected ClusterTestBase(IClusterFixture clusterFixture)
_runId = Guid.NewGuid().ToString("N").Substring(0, 6);
}

protected Task Trace(Func<Task> test, ITestOutputHelper output, [CallerMemberName]string testName = "")
{
return ClusterFixture.Trace(test, testName);
}

protected LogStore LogStore => ClusterFixture.LogStore;

protected IList<Cluster> Members => ClusterFixture.Members;
Expand Down
32 changes: 16 additions & 16 deletions tests/Proto.Cluster.Tests/ClusterTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void ClusterMembersMatch()
[Fact]
public async Task CanSpawnASingleVirtualActor()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token;
Expand All @@ -55,7 +55,7 @@ await Tracing.Trace(async () =>
[Fact]
public async Task TopologiesShouldHaveConsensus()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var consensus = await Task
.WhenAll(Members.Select(member =>
Expand All @@ -73,7 +73,7 @@ await Tracing.Trace(async () =>
[Fact]
public async Task HandlesSlowResponsesCorrectly()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(20000).Token;
Expand All @@ -92,7 +92,7 @@ await Tracing.Trace(async () =>
[Fact]
public async Task SupportsMessageEnvelopeResponses()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(20000).Token;
Expand All @@ -113,7 +113,7 @@ await Tracing.Trace(async () =>
[Fact]
public async Task StateIsReplicatedAcrossCluster()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
if (ClusterFixture.ClusterSize < 2)
{
Expand Down Expand Up @@ -165,7 +165,7 @@ IAsyncEnumerable<GossipUpdate> SubscribeToGossipUpdates(Cluster member)
[Fact]
public async Task ReSpawnsClusterActorsFromDifferentNodes()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
if (ClusterFixture.ClusterSize < 2)
{
Expand Down Expand Up @@ -200,7 +200,7 @@ await Tracing.Trace(async () =>
[Fact]
public async Task HandlesLosingANode()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
if (ClusterFixture.ClusterSize < 2)
{
Expand Down Expand Up @@ -228,7 +228,7 @@ await Tracing.Trace(async () =>
[Fact]
public async Task HandlesLosingANodeWhileProcessing()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
if (ClusterFixture.ClusterSize < 2)
{
Expand Down Expand Up @@ -276,7 +276,7 @@ private async Task CanGetResponseFromAllIdsOnAllNodes(IEnumerable<string> actorI
[InlineData(10, 10000)]
public async Task CanSpawnVirtualActorsSequentially(int actorCount, int timeoutMs)
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(timeoutMs).Token;
Expand All @@ -298,7 +298,7 @@ await Tracing.Trace(async () =>
[InlineData(10, 10000)]
public async Task ConcurrentActivationsOnSameIdWorks(int clientCount, int timeoutMs)
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(timeoutMs).Token;
Expand All @@ -318,7 +318,7 @@ await Tracing.Trace(async () =>
[InlineData(10, 10000)]
public async Task CanSpawnVirtualActorsConcurrently(int actorCount, int timeoutMs)
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(timeoutMs).Token;
Expand All @@ -335,7 +335,7 @@ await Tracing.Trace(async () =>
[InlineData(10, 10000)]
public async Task CanSpawnMultipleKindsWithSameIdentityConcurrently(int actorCount, int timeoutMs)
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
using var cts = new CancellationTokenSource(timeoutMs);
var timeout = cts.Token;
Expand Down Expand Up @@ -364,7 +364,7 @@ await Task.WhenAll(actorIds.Select(id => Task.WhenAll(
[InlineData(10, 10000)]
public async Task CanSpawnMultipleKindsWithSameIdentityConcurrentlyWhenUsingFilters(int actorCount, int timeoutMs)
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
using var cts = new CancellationTokenSource(timeoutMs);
var timeout = cts.Token;
Expand Down Expand Up @@ -395,7 +395,7 @@ await Task.WhenAll(actorIds.Select(id => Task.WhenAll(
[InlineData(10, 10000, EchoActor.AsyncFilteredKind)]
public async Task CanSpawnVirtualActorsConcurrentlyOnAllNodes(int actorCount, int timeoutMs, string kind)
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(timeoutMs).Token;
Expand All @@ -415,7 +415,7 @@ await Tracing.Trace(async () =>
[InlineData(10000, EchoActor.FilteredKind)]
[InlineData(10000, EchoActor.AsyncFilteredKind)]
public async Task CanFilterActivations(int timeoutMs, string filteredKind) =>
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(timeoutMs).Token;
Expand All @@ -436,7 +436,7 @@ await member.Invoking(async m => await m.RequestAsync<Pong>(invalidIdentity, mes
[InlineData(10, 20000)]
public async Task CanRespawnVirtualActors(int actorCount, int timeoutMs)
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
using var cts = new CancellationTokenSource(timeoutMs);
var timeout = cts.Token;
Expand Down
2 changes: 1 addition & 1 deletion tests/Proto.Cluster.Tests/ClusterTestsWithLocalAffinity.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ protected ClusterTestsWithLocalAffinity(ITestOutputHelper testOutputHelper, IClu
[Fact]
public async Task LocalAffinityMovesActivationsOnRemoteSender()
{
await Tracing.Trace(async () =>
await Trace(async () =>
{
var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(20)).Token;
var firstNode = Members[0];
Expand Down
130 changes: 130 additions & 0 deletions tests/Proto.Cluster.Tests/GithubActionsReporter.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;

namespace Proto.Cluster.Tests;

public class GithubActionsReporter
{
private readonly string _reportName;
private static readonly ILogger Logger = Log.CreateLogger<GithubActionsReporter>();
public const string ActivitySourceName = "Proto.Cluster.Tests";

private static readonly ActivitySource ActivitySource = new(ActivitySourceName);

public GithubActionsReporter(string reportName)
{
_reportName = reportName;
}

private static Activity? StartActivity([CallerMemberName] string callerName = "N/A") =>
ActivitySource.StartActivity(callerName);

private List<TestResult> _results = new();

private record TestResult(string Name, string TraceId, TimeSpan Duration, Exception? Exception= null);

private readonly StringBuilder _output = new();

public async Task Run(Func<Task> test, [CallerMemberName]string testName="")
{
await Task.Delay(1).ConfigureAwait(false);

using var activity = StartActivity(testName);
var traceId= activity?.Context.TraceId.ToString().ToUpperInvariant() ?? "N/A";
Logger.LogInformation("Test started");
Exception? exception = null;
var sw = Stopwatch.StartNew();

if (activity is not null)
{
traceId = activity.TraceId.ToString();
activity.AddTag("test.name", testName);

var traceViewUrl =
$"{TracingSettings.TraceViewUrl}/logs?traceId={traceId}";

Console.WriteLine($"Running test: {testName}");
Console.WriteLine(traceViewUrl);
}

try
{
await test();
Logger.LogInformation("Test succeeded");
}
catch(Exception x)
{
exception = x;
Logger.LogError(x,"Test failed");
}
sw.Stop();
if (activity is not null)
{
_results.Add(new TestResult(testName, traceId, sw.Elapsed, exception));
}
}

public async Task WriteReportFile()
{
var failIcon =
"<img src=\"https://gist.githubusercontent.com/rogeralsing/d8566b01e0850be70f7af9bc9757691e/raw/e025b5d58fe3aec1029a5c74f5ab2ee198960fcb/fail.svg\">";
var successIcon =
"<img src=\"https://gist.githubusercontent.com/rogeralsing/b9165f8eaeb25f05226745c94ab011b6/raw/cb28ccf1a11c44c8b4c9173bc4aeb98bfa79ca4b/success.svg\">";

var serverUrl = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL");
var repositorySlug = Environment.GetEnvironmentVariable("GITHUB_REPOSITORY");
var workspacePath = Environment.GetEnvironmentVariable("GITHUB_WORKSPACE");
var commitHash = Environment.GetEnvironmentVariable("GITHUB_SHA");
var f = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY");
if (f != null)
{
_output.AppendLine($@"
<h2>{_reportName}</h2>
<table>
<tr>
<th>
Test
</th>
<th>
Duration
</th>
</tr>");

foreach (var res in _results)
{
_output.AppendLine($@"
<tr>
<td>
{(res.Exception != null ? failIcon : successIcon)}
<a href=""{TracingSettings.TraceViewUrl}/logs?traceId={res.TraceId}"">{res.Name}</a>
</td>
<td>
{res.Duration}
</td>
</tr>");
if(res.Exception is not null)
{
_output.AppendLine($@"
<tr>
<td colspan=""2"">
<code>
{res.Exception}
</code>
</td>
</tr>");
}
}
_output.AppendLine("</table>");

await File.AppendAllTextAsync(f, _output.ToString());
}
}
}
Loading

0 comments on commit 2107017

Please sign in to comment.