diff --git a/tests/Proto.Cluster.Tests/ClusterFixture.cs b/tests/Proto.Cluster.Tests/ClusterFixture.cs index 3c94191c4f..3da740154b 100644 --- a/tests/Proto.Cluster.Tests/ClusterFixture.cs +++ b/tests/Proto.Cluster.Tests/ClusterFixture.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using ClusterTest.Messages; @@ -38,6 +39,8 @@ public interface IClusterFixture public Task SpawnNode(); Task RemoveNode(Cluster member, bool graceful = true); + + Task Trace(Func test, [CallerMemberName] string testName = ""); } public static class TracingSettings @@ -57,6 +60,7 @@ public abstract class ClusterFixture : IAsyncLifetime, IClusterFixture, IAsyncDi private readonly ILogger _logger = Log.CreateLogger(nameof(GetType)); private readonly List _members = new(); private static TracerProvider? _tracerProvider; + private GithubActionsReporter _reporter; protected readonly string ClusterName; @@ -65,13 +69,14 @@ static ClusterFixture() TracingSettings.OpenTelemetryUrl = Environment.GetEnvironmentVariable("OPENTELEMETRY_URL"); TracingSettings.TraceViewUrl = Environment.GetEnvironmentVariable("TRACEVIEW_URL"); TracingSettings.EnableTracing = TracingSettings.OpenTelemetryUrl != null; - + //TODO: check if this helps low resource envs like github actions. ThreadPool.SetMinThreads(40, 40); } - protected ClusterFixture(int clusterSize, Func? configure = null) + protected ClusterFixture( int clusterSize, Func? configure = null) { + _reporter = new GithubActionsReporter(GetType().Name); #if NETCOREAPP3_1 AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); #endif @@ -117,22 +122,11 @@ public async Task DisposeAsync() { try { + await _reporter.WriteReportFile(); await OnDisposing(); - if (TracingSettings.EnableTracing) - { - var testName = this.GetType().Name; - using (Tracing.StartActivity("ClusterFixture.DisposeAsync " + testName)) - { - await WaitForMembersToShutdown(); - } - _tracerProvider?.ForceFlush(1000); - } - else - { - await WaitForMembersToShutdown(); - } + await WaitForMembersToShutdown(); Members.Clear(); // prevent multiple shutdown attempts if dispose is called multiple times } @@ -189,6 +183,11 @@ public async Task RemoveNode(Cluster member, bool graceful = true) } } + public Task Trace(Func test, string testName = "") + { + return _reporter.Run(test, testName); + } + /// /// Spawns a node, adds it to the cluster and member list /// @@ -243,7 +242,7 @@ private static void InitOpenTelemetryTracing() .AddService("Proto.Cluster.Tests") ) .AddProtoActorInstrumentation() - .AddSource(Tracing.ActivitySourceName) + .AddSource(GithubActionsReporter.ActivitySourceName) .AddOtlpExporter(options => { options.Endpoint = endpoint; diff --git a/tests/Proto.Cluster.Tests/ClusterTestBase.cs b/tests/Proto.Cluster.Tests/ClusterTestBase.cs index 65c78dee01..9ab06cb063 100644 --- a/tests/Proto.Cluster.Tests/ClusterTestBase.cs +++ b/tests/Proto.Cluster.Tests/ClusterTestBase.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; using Proto.Logging; using Xunit; +using Xunit.Abstractions; namespace Proto.Cluster.Tests; @@ -18,6 +21,11 @@ protected ClusterTestBase(IClusterFixture clusterFixture) _runId = Guid.NewGuid().ToString("N").Substring(0, 6); } + protected Task Trace(Func test, ITestOutputHelper output, [CallerMemberName]string testName = "") + { + return ClusterFixture.Trace(test, testName); + } + protected LogStore LogStore => ClusterFixture.LogStore; protected IList Members => ClusterFixture.Members; diff --git a/tests/Proto.Cluster.Tests/ClusterTests.cs b/tests/Proto.Cluster.Tests/ClusterTests.cs index 6deb5b0177..b562b633ee 100644 --- a/tests/Proto.Cluster.Tests/ClusterTests.cs +++ b/tests/Proto.Cluster.Tests/ClusterTests.cs @@ -39,7 +39,7 @@ public void ClusterMembersMatch() [Fact] public async Task CanSpawnASingleVirtualActor() { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token; @@ -55,7 +55,7 @@ await Tracing.Trace(async () => [Fact] public async Task TopologiesShouldHaveConsensus() { - await Tracing.Trace(async () => + await Trace(async () => { var consensus = await Task .WhenAll(Members.Select(member => @@ -73,7 +73,7 @@ await Tracing.Trace(async () => [Fact] public async Task HandlesSlowResponsesCorrectly() { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(20000).Token; @@ -92,7 +92,7 @@ await Tracing.Trace(async () => [Fact] public async Task SupportsMessageEnvelopeResponses() { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(20000).Token; @@ -113,7 +113,7 @@ await Tracing.Trace(async () => [Fact] public async Task StateIsReplicatedAcrossCluster() { - await Tracing.Trace(async () => + await Trace(async () => { if (ClusterFixture.ClusterSize < 2) { @@ -165,7 +165,7 @@ IAsyncEnumerable SubscribeToGossipUpdates(Cluster member) [Fact] public async Task ReSpawnsClusterActorsFromDifferentNodes() { - await Tracing.Trace(async () => + await Trace(async () => { if (ClusterFixture.ClusterSize < 2) { @@ -200,7 +200,7 @@ await Tracing.Trace(async () => [Fact] public async Task HandlesLosingANode() { - await Tracing.Trace(async () => + await Trace(async () => { if (ClusterFixture.ClusterSize < 2) { @@ -228,7 +228,7 @@ await Tracing.Trace(async () => [Fact] public async Task HandlesLosingANodeWhileProcessing() { - await Tracing.Trace(async () => + await Trace(async () => { if (ClusterFixture.ClusterSize < 2) { @@ -276,7 +276,7 @@ private async Task CanGetResponseFromAllIdsOnAllNodes(IEnumerable actorI [InlineData(10, 10000)] public async Task CanSpawnVirtualActorsSequentially(int actorCount, int timeoutMs) { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(timeoutMs).Token; @@ -298,7 +298,7 @@ await Tracing.Trace(async () => [InlineData(10, 10000)] public async Task ConcurrentActivationsOnSameIdWorks(int clientCount, int timeoutMs) { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(timeoutMs).Token; @@ -318,7 +318,7 @@ await Tracing.Trace(async () => [InlineData(10, 10000)] public async Task CanSpawnVirtualActorsConcurrently(int actorCount, int timeoutMs) { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(timeoutMs).Token; @@ -335,7 +335,7 @@ await Tracing.Trace(async () => [InlineData(10, 10000)] public async Task CanSpawnMultipleKindsWithSameIdentityConcurrently(int actorCount, int timeoutMs) { - await Tracing.Trace(async () => + await Trace(async () => { using var cts = new CancellationTokenSource(timeoutMs); var timeout = cts.Token; @@ -364,7 +364,7 @@ await Task.WhenAll(actorIds.Select(id => Task.WhenAll( [InlineData(10, 10000)] public async Task CanSpawnMultipleKindsWithSameIdentityConcurrentlyWhenUsingFilters(int actorCount, int timeoutMs) { - await Tracing.Trace(async () => + await Trace(async () => { using var cts = new CancellationTokenSource(timeoutMs); var timeout = cts.Token; @@ -395,7 +395,7 @@ await Task.WhenAll(actorIds.Select(id => Task.WhenAll( [InlineData(10, 10000, EchoActor.AsyncFilteredKind)] public async Task CanSpawnVirtualActorsConcurrentlyOnAllNodes(int actorCount, int timeoutMs, string kind) { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(timeoutMs).Token; @@ -415,7 +415,7 @@ await Tracing.Trace(async () => [InlineData(10000, EchoActor.FilteredKind)] [InlineData(10000, EchoActor.AsyncFilteredKind)] public async Task CanFilterActivations(int timeoutMs, string filteredKind) => - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(timeoutMs).Token; @@ -436,7 +436,7 @@ await member.Invoking(async m => await m.RequestAsync(invalidIdentity, mes [InlineData(10, 20000)] public async Task CanRespawnVirtualActors(int actorCount, int timeoutMs) { - await Tracing.Trace(async () => + await Trace(async () => { using var cts = new CancellationTokenSource(timeoutMs); var timeout = cts.Token; diff --git a/tests/Proto.Cluster.Tests/ClusterTestsWithLocalAffinity.cs b/tests/Proto.Cluster.Tests/ClusterTestsWithLocalAffinity.cs index 0b67daa78f..116c0ca64b 100644 --- a/tests/Proto.Cluster.Tests/ClusterTestsWithLocalAffinity.cs +++ b/tests/Proto.Cluster.Tests/ClusterTestsWithLocalAffinity.cs @@ -19,7 +19,7 @@ protected ClusterTestsWithLocalAffinity(ITestOutputHelper testOutputHelper, IClu [Fact] public async Task LocalAffinityMovesActivationsOnRemoteSender() { - await Tracing.Trace(async () => + await Trace(async () => { var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(20)).Token; var firstNode = Members[0]; diff --git a/tests/Proto.Cluster.Tests/GithubActionsReporter.cs b/tests/Proto.Cluster.Tests/GithubActionsReporter.cs new file mode 100644 index 0000000000..4f082fe718 --- /dev/null +++ b/tests/Proto.Cluster.Tests/GithubActionsReporter.cs @@ -0,0 +1,130 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Proto.Cluster.Tests; + +public class GithubActionsReporter +{ + private readonly string _reportName; + private static readonly ILogger Logger = Log.CreateLogger(); + public const string ActivitySourceName = "Proto.Cluster.Tests"; + + private static readonly ActivitySource ActivitySource = new(ActivitySourceName); + + public GithubActionsReporter(string reportName) + { + _reportName = reportName; + } + + private static Activity? StartActivity([CallerMemberName] string callerName = "N/A") => + ActivitySource.StartActivity(callerName); + + private List _results = new(); + + private record TestResult(string Name, string TraceId, TimeSpan Duration, Exception? Exception= null); + + private readonly StringBuilder _output = new(); + + public async Task Run(Func test, [CallerMemberName]string testName="") + { + await Task.Delay(1).ConfigureAwait(false); + + using var activity = StartActivity(testName); + var traceId= activity?.Context.TraceId.ToString().ToUpperInvariant() ?? "N/A"; + Logger.LogInformation("Test started"); + Exception? exception = null; + var sw = Stopwatch.StartNew(); + + if (activity is not null) + { + traceId = activity.TraceId.ToString(); + activity.AddTag("test.name", testName); + + var traceViewUrl = + $"{TracingSettings.TraceViewUrl}/logs?traceId={traceId}"; + + Console.WriteLine($"Running test: {testName}"); + Console.WriteLine(traceViewUrl); + } + + try + { + await test(); + Logger.LogInformation("Test succeeded"); + } + catch(Exception x) + { + exception = x; + Logger.LogError(x,"Test failed"); + } + sw.Stop(); + if (activity is not null) + { + _results.Add(new TestResult(testName, traceId, sw.Elapsed, exception)); + } + } + + public async Task WriteReportFile() + { + var failIcon = + ""; + var successIcon = + ""; + + var serverUrl = Environment.GetEnvironmentVariable("GITHUB_SERVER_URL"); + var repositorySlug = Environment.GetEnvironmentVariable("GITHUB_REPOSITORY"); + var workspacePath = Environment.GetEnvironmentVariable("GITHUB_WORKSPACE"); + var commitHash = Environment.GetEnvironmentVariable("GITHUB_SHA"); + var f = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY"); + if (f != null) + { + _output.AppendLine($@" +

{_reportName}

+ + + + + +"); + + foreach (var res in _results) + { + _output.AppendLine($@" + + + +"); + if(res.Exception is not null) + { + _output.AppendLine($@" + + +"); + } + } + _output.AppendLine("
+Test + +Duration +
+{(res.Exception != null ? failIcon : successIcon)} +{res.Name} + +{res.Duration} +
+ +{res.Exception} + +
"); + + await File.AppendAllTextAsync(f, _output.ToString()); + } + } +} \ No newline at end of file diff --git a/tests/Proto.Cluster.Tests/GossipTests.cs b/tests/Proto.Cluster.Tests/GossipTests.cs index f690e64d59..6ada22a9c1 100644 --- a/tests/Proto.Cluster.Tests/GossipTests.cs +++ b/tests/Proto.Cluster.Tests/GossipTests.cs @@ -55,41 +55,39 @@ public async Task CompositeConsensusWorks() var timeout = CancellationTokens.FromSeconds(20); await using var clusterFixture = new InMemoryClusterFixture(); await clusterFixture.InitializeAsync().ConfigureAwait(false); - await Tracing.Trace(async () => - { - await Task.Delay(1000); + await Task.Delay(1000); + + var (consensus, initialTopologyHash) = + await clusterFixture.Members.First().MemberList.TopologyConsensus(timeout); - var (consensus, initialTopologyHash) = - await clusterFixture.Members.First().MemberList.TopologyConsensus(timeout); + consensus.Should().BeTrue(); - consensus.Should().BeTrue(); + var fixtureMembers = clusterFixture.Members; + var consensusChecks = fixtureMembers.Select(CreateCompositeConsensusCheck).ToList(); - var fixtureMembers = clusterFixture.Members; - var consensusChecks = fixtureMembers.Select(CreateCompositeConsensusCheck).ToList(); + var firstNodeCheck = consensusChecks[0]; + var notConsensus = await firstNodeCheck.TryGetConsensus(TimeSpan.FromMilliseconds(200), timeout); - var firstNodeCheck = consensusChecks[0]; - var notConsensus = await firstNodeCheck.TryGetConsensus(TimeSpan.FromMilliseconds(200), timeout); + notConsensus.consensus.Should().BeFalse("We have not set the correct topology hash in the state yet"); - notConsensus.consensus.Should().BeFalse("We have not set the correct topology hash in the state yet"); + await SetTopologyGossipStateAsync(fixtureMembers, initialTopologyHash); - await SetTopologyGossipStateAsync(fixtureMembers, initialTopologyHash); + var afterSettingMatchingState = await firstNodeCheck.TryGetConsensus(TimeSpan.FromSeconds(20), timeout); - var afterSettingMatchingState = await firstNodeCheck.TryGetConsensus(TimeSpan.FromSeconds(20), timeout); + afterSettingMatchingState.consensus.Should() + .BeTrue("After assigning the matching topology hash, there should be consensus"); - afterSettingMatchingState.consensus.Should() - .BeTrue("After assigning the matching topology hash, there should be consensus"); + afterSettingMatchingState.value.Should().Be(initialTopologyHash); - afterSettingMatchingState.value.Should().Be(initialTopologyHash); + await clusterFixture.SpawnNode(); + await Task.Delay(2000); // Allow topology state to propagate - await clusterFixture.SpawnNode(); - await Task.Delay(2000); // Allow topology state to propagate + var afterChangingTopology = + await firstNodeCheck.TryGetConsensus(TimeSpan.FromMilliseconds(500), timeout); - var afterChangingTopology = - await firstNodeCheck.TryGetConsensus(TimeSpan.FromMilliseconds(500), timeout); + afterChangingTopology.consensus.Should().BeFalse("The state does no longer match the current topology"); - afterChangingTopology.consensus.Should().BeFalse("The state does no longer match the current topology"); - }, _testOutputHelper); } [Fact] diff --git a/tests/Proto.Cluster.Tests/Tracing.cs b/tests/Proto.Cluster.Tests/Tracing.cs deleted file mode 100644 index feab5a6d26..0000000000 --- a/tests/Proto.Cluster.Tests/Tracing.cs +++ /dev/null @@ -1,102 +0,0 @@ -// ----------------------------------------------------------------------- -// -// Copyright (C) 2015-2022 Asynkron AB All rights reserved -// -// ----------------------------------------------------------------------- - -using System; -using System.Diagnostics; -using System.IO; -using System.Runtime.CompilerServices; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using OpenTelemetry.Trace; -using Proto.Utils; -using Xunit.Abstractions; - -namespace Proto.Cluster.Tests; - - -public static class Tracing -{ - public const string ActivitySourceName = "Proto.Cluster.Tests"; - - public static readonly ActivitySource ActivitySource = new(ActivitySourceName); - - public static Activity StartActivity([CallerMemberName] string callerName = "N/A") => - ActivitySource.StartActivity(callerName); - - public static async Task Trace(Func callBack, ITestOutputHelper testOutputHelper, - [CallerMemberName] string callerName = "N/A") - { - await Task.Delay(1).ConfigureAwait(false); - var logger = Log.CreateLogger(callerName); - using var activity = StartActivity(callerName); - logger.LogInformation("Test started"); - var traceId = ""; - var success = true; - var error = ""; - var sw = Stopwatch.StartNew(); - - if (activity is not null) - { - traceId = activity.TraceId.ToString(); - activity.AddTag("test.name", callerName); - - var traceViewUrl = - $"{TracingSettings.TraceViewUrl}/logs?traceId={activity.TraceId.ToString().ToUpperInvariant()}"; - - testOutputHelper.WriteLine(traceViewUrl); - Console.WriteLine($"Running test: {callerName}"); - Console.WriteLine(traceViewUrl); - } - else - { - testOutputHelper.WriteLine("No active trace span"); - } - - try - { - var res = await callBack().WaitUpTo(TimeSpan.FromSeconds(30)); - if (!res) - { - testOutputHelper.WriteLine($"{callerName} timedout"); - throw new TimeoutException($"{callerName} timedout"); - } - } - catch (Exception e) - { - activity?.SetStatus(ActivityStatusCode.Error); - activity?.RecordException(e); - error = e.ToString(); - success = false; - throw; - } - finally - { - logger.LogInformation("Test ended"); - - var f = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY"); - if (f != null && traceId != "") - { - var traceViewUrl = - $"{TracingSettings.TraceViewUrl}/logs?traceId={traceId.ToUpperInvariant()}"; - - var duration = sw.Elapsed; - var failIcon = - ""; - var successIcon = - ""; - - - - var markdown = $@" -{(success ? successIcon : failIcon)} [Test: {callerName}]({traceViewUrl}) - Duration: {duration.TotalMilliseconds} ms
-{(success ? "" : $"Error:\n```\n{error}\n```")} -"; - await File.AppendAllTextAsync(f, markdown); - - } - } - } -} \ No newline at end of file