Skip to content

Commit 9f6f0d9

Browse files
committed
Make in-process diagnosers work in wasm and in-process toolchains.
Add tests.
1 parent c52ac06 commit 9f6f0d9

File tree

12 files changed

+251
-58
lines changed

12 files changed

+251
-58
lines changed

src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
6161
public sealed class CompositeInProcessDiagnoser(IReadOnlyList<IInProcessDiagnoser> inProcessDiagnosers)
6262
{
6363
public const string HeaderKey = "// InProcessDiagnoser";
64-
public const string ResultsKey = "// InProcessDiagnoserResults";
64+
public const string ResultsKey = $"{HeaderKey}Results";
6565

6666
public IEnumerable<string> GetHandlersSourceCode(BenchmarkCase benchmarkCase)
6767
=> inProcessDiagnosers

src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,10 @@ internal static ClrMdDisassembler GetClrMdDisassembler() =>
4545

4646
private readonly MonoDisassembler monoDisassembler = new();
4747
private readonly Dictionary<BenchmarkCase, DisassemblyResult> results = [];
48-
private readonly bool runInHost;
4948

5049
public DisassemblyDiagnoser(DisassemblyDiagnoserConfig config)
5150
{
5251
Config = config;
53-
runInHost = Config.RunInHost;
5452

5553
Exporters = GetExporters(results, config);
5654
}
@@ -100,7 +98,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
10098

10199
switch (signal)
102100
{
103-
case HostSignal.AfterAll when (runInHost || isInProcess) && ShouldUseClrMdDisassembler(benchmark):
101+
case HostSignal.AfterAll when (Config.RunInHost || isInProcess) && ShouldUseClrMdDisassembler(benchmark):
104102
results.Add(benchmark, ClrMdDisassembler.AttachAndDisassemble(
105103
BuildDisassemblerSettings(parameters.BenchmarkCase, $"BenchmarkDotNet.Autogenerated.Runnable_{parameters.BenchmarkId.Value}", parameters.Process.Id))
106104
);
@@ -139,7 +137,7 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
139137

140138
if (ShouldUseClrMdDisassembler(benchmark))
141139
{
142-
if (runInHost && toolchain?.IsInProcess != true && !PlatformsMatch(currentPlatform, benchmark.Job.Environment.Platform))
140+
if (Config.RunInHost && toolchain?.IsInProcess != true && !PlatformsMatch(currentPlatform, benchmark.Job.Environment.Platform))
143141
{
144142
yield return new ValidationError(true, "DisassemblyDiagnoser cannot run in host for a job that targets a different platform", benchmark);
145143
}
@@ -216,7 +214,7 @@ private static long SumNativeCodeSize(DisassemblyResult disassembly)
216214
string IInProcessDiagnoser.GetHandlerSourceCode(BenchmarkCase benchmarkCase, int index)
217215
{
218216
// Mono disassembler always runs another process.
219-
if (runInHost || ShouldUseMonoDisassembler(benchmarkCase))
217+
if (Config.RunInHost || ShouldUseMonoDisassembler(benchmarkCase))
220218
{
221219
return null;
222220
}

src/BenchmarkDotNet/Loggers/Broker.cs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,17 +90,23 @@ private void ProcessDataBlocking()
9090
{
9191
Results.Add(line);
9292
}
93+
// Keep in sync with WasmExecutor and InProcessHost.
9394
else if (line.StartsWith(CompositeInProcessDiagnoser.HeaderKey))
9495
{
95-
// Something like "// InProcessDiagnoserHandlerResults 0 1"
96+
// Something like "// InProcessDiagnoser 0 1"
9697
string[] lineItems = line.Split(' ');
9798
int diagnoserIndex = int.Parse(lineItems[2]);
9899
int resultsLinesCount = int.Parse(lineItems[3]);
99100
var resultsStringBuilder = new StringBuilder();
100-
for (int i = 0; i < resultsLinesCount; ++i)
101+
for (int i = 0; i < resultsLinesCount;)
101102
{
102103
// Strip the prepended "// InProcessDiagnoserResults ".
103-
resultsStringBuilder.AppendLine(reader.ReadLine().Substring(CompositeInProcessDiagnoser.ResultsKey.Length + 1));
104+
line = reader.ReadLine().Substring(CompositeInProcessDiagnoser.ResultsKey.Length + 1);
105+
resultsStringBuilder.Append(line);
106+
if (++i < resultsLinesCount)
107+
{
108+
resultsStringBuilder.AppendLine();
109+
}
104110
}
105111
compositeInProcessDiagnoser.DeserializeResults(diagnoserIndex, DiagnoserActionParameters.BenchmarkCase, resultsStringBuilder.ToString());
106112
}

src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ public ExecuteResult Execute(ExecuteParameters executeParameters)
7676
$"Benchmark {executeParameters.BenchmarkCase.DisplayInfo} takes too long to run. " +
7777
"Prefer to use out-of-process toolchains for long-running benchmarks.");
7878

79+
host.HandleInProcessDiagnoserResults(executeParameters.BenchmarkCase, executeParameters.CompositeInProcessDiagnoser);
80+
7981
return ExecuteResult.FromRunResults(host.RunResults, exitCode);
8082
}
8183

src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs

Lines changed: 42 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,25 @@
11
using System;
2+
using System.Collections.Generic;
23
using System.Diagnostics;
34
using System.IO;
4-
5+
using System.Text;
56
using BenchmarkDotNet.Configs;
67
using BenchmarkDotNet.Diagnosers;
78
using BenchmarkDotNet.Engines;
89
using BenchmarkDotNet.Loggers;
910
using BenchmarkDotNet.Running;
1011
using BenchmarkDotNet.Validators;
1112

12-
using JetBrains.Annotations;
13-
1413
namespace BenchmarkDotNet.Toolchains.InProcess
1514
{
1615
/// <summary>Host API for in-process benchmarks.</summary>
1716
/// <seealso cref="IHost"/>
18-
public sealed class InProcessHost : IHost
17+
internal sealed class InProcessHost : IHost
1918
{
2019
private readonly ILogger logger;
21-
2220
private readonly IDiagnoser? diagnoser;
23-
2421
private readonly DiagnoserActionParameters? diagnoserActionParameters;
22+
private readonly List<string> inProcessDiagnoserLines = [];
2523

2624
/// <summary>Creates a new instance of <see cref="InProcessHost"/>.</summary>
2725
/// <param name="benchmarkCase">Current benchmark.</param>
@@ -34,7 +32,6 @@ public InProcessHost(BenchmarkCase benchmarkCase, ILogger logger, IDiagnoser dia
3432

3533
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
3634
this.diagnoser = diagnoser;
37-
IsDiagnoserAttached = diagnoser != null;
3835
Config = benchmarkCase.Config;
3936

4037
if (diagnoser != null)
@@ -44,16 +41,12 @@ public InProcessHost(BenchmarkCase benchmarkCase, ILogger logger, IDiagnoser dia
4441
default);
4542
}
4643

47-
/// <summary><c>True</c> if there are diagnosers attached.</summary>
48-
/// <value><c>True</c> if there are diagnosers attached.</value>
49-
[PublicAPI] public bool IsDiagnoserAttached { get; }
50-
5144
/// <summary>Results of the run.</summary>
5245
/// <value>Results of the run.</value>
5346
public RunResults RunResults { get; private set; }
5447

5548
/// <summary>Current config</summary>
56-
[PublicAPI] public IConfig Config { get; set; }
49+
public IConfig Config { get; set; }
5750

5851
/// <summary>Passes text to the host.</summary>
5952
/// <param name="message">Text to write.</param>
@@ -64,20 +57,18 @@ public InProcessHost(BenchmarkCase benchmarkCase, ILogger logger, IDiagnoser dia
6457

6558
/// <summary>Passes text (new line appended) to the host.</summary>
6659
/// <param name="message">Text to write.</param>
67-
public void WriteLine(string message) => logger.WriteLine(message);
60+
public void WriteLine(string message)
61+
{
62+
logger.WriteLine(message);
63+
if (message.StartsWith(CompositeInProcessDiagnoser.HeaderKey)) // Captures both header and results
64+
{
65+
inProcessDiagnoserLines.Add(message);
66+
}
67+
}
6868

6969
/// <summary>Sends notification signal to the host.</summary>
7070
/// <param name="hostSignal">The signal to send.</param>
71-
public void SendSignal(HostSignal hostSignal)
72-
{
73-
if (!IsDiagnoserAttached) // no need to send the signal, nobody is listening for it
74-
return;
75-
76-
if (diagnoser == null)
77-
throw new NullReferenceException(nameof(diagnoser));
78-
79-
diagnoser.Handle(hostSignal, diagnoserActionParameters);
80-
}
71+
public void SendSignal(HostSignal hostSignal) => diagnoser?.Handle(hostSignal, diagnoserActionParameters);
8172

8273
public void SendError(string message) => logger.WriteLine(LogKind.Error, $"{ValidationErrorReporter.ConsoleErrorPrefix} {message}");
8374

@@ -94,6 +85,34 @@ public void ReportResults(RunResults runResults)
9485
}
9586
}
9687

88+
// Keep in sync with Broker and WasmExecutor.
89+
internal void HandleInProcessDiagnoserResults(BenchmarkCase benchmarkCase, CompositeInProcessDiagnoser compositeInProcessDiagnoser)
90+
{
91+
var linesEnumerator = inProcessDiagnoserLines.GetEnumerator();
92+
while (linesEnumerator.MoveNext())
93+
{
94+
// Something like "// InProcessDiagnoser 0 1"
95+
var line = linesEnumerator.Current;
96+
string[] lineItems = line.Split(' ');
97+
int diagnoserIndex = int.Parse(lineItems[2]);
98+
int resultsLinesCount = int.Parse(lineItems[3]);
99+
var resultsStringBuilder = new StringBuilder();
100+
for (int i = 0; i < resultsLinesCount;)
101+
{
102+
// Strip the prepended "// InProcessDiagnoserResults ".
103+
bool movedNext = linesEnumerator.MoveNext();
104+
Debug.Assert(movedNext);
105+
line = linesEnumerator.Current.Substring(CompositeInProcessDiagnoser.ResultsKey.Length + 1);
106+
resultsStringBuilder.Append(line);
107+
if (++i < resultsLinesCount)
108+
{
109+
resultsStringBuilder.AppendLine();
110+
}
111+
}
112+
compositeInProcessDiagnoser.DeserializeResults(diagnoserIndex, benchmarkCase, resultsStringBuilder.ToString());
113+
}
114+
}
115+
97116
public void Dispose()
98117
{
99118
// do nothing on purpose

src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ public ExecuteResult Execute(ExecuteParameters executeParameters)
7676
$"Benchmark {executeParameters.BenchmarkCase.DisplayInfo} takes too long to run. " +
7777
"Prefer to use out-of-process toolchains for long-running benchmarks.");
7878

79+
host.HandleInProcessDiagnoserResults(executeParameters.BenchmarkCase, executeParameters.CompositeInProcessDiagnoser);
80+
7981
return ExecuteResult.FromRunResults(host.RunResults, exitCode);
8082
}
8183

src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@
1010
using BenchmarkDotNet.Toolchains.Parameters;
1111
using BenchmarkDotNet.Toolchains.Results;
1212
using System;
13+
using System.Collections.Generic;
1314
using System.Collections.Immutable;
1415
using System.Diagnostics;
1516
using System.IO;
16-
using System.Linq;
17+
using System.Text;
1718

1819
namespace BenchmarkDotNet.Toolchains.MonoWasm
1920
{
@@ -100,14 +101,50 @@ private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCas
100101
processOutputReader.StopRead();
101102
}
102103

103-
// TODO: Handle InProcessDiagnoser messages since this executor does not use Broker.
104104
ImmutableArray<string> outputLines = processOutputReader.GetOutputLines();
105+
var prefixedLines = new List<string>();
106+
var standardOutput = new List<string>();
107+
var outputEnumerator = outputLines.GetEnumerator();
108+
while (outputEnumerator.MoveNext())
109+
{
110+
var line = outputEnumerator.Current;
111+
if (!line.StartsWith("//"))
112+
{
113+
standardOutput.Add(line);
114+
continue;
115+
}
116+
117+
prefixedLines.Add(line);
118+
119+
// Keep in sync with Broker and InProcessHost.
120+
if (line.StartsWith(CompositeInProcessDiagnoser.HeaderKey))
121+
{
122+
// Something like "// InProcessDiagnoser 0 1"
123+
string[] lineItems = line.Split(' ');
124+
int diagnoserIndex = int.Parse(lineItems[2]);
125+
int resultsLinesCount = int.Parse(lineItems[3]);
126+
var resultsStringBuilder = new StringBuilder();
127+
for (int i = 0; i < resultsLinesCount;)
128+
{
129+
// Strip the prepended "// InProcessDiagnoserResults ".
130+
bool movedNext = outputEnumerator.MoveNext();
131+
Debug.Assert(movedNext);
132+
line = outputEnumerator.Current.Substring(CompositeInProcessDiagnoser.ResultsKey.Length + 1);
133+
resultsStringBuilder.Append(line);
134+
if (++i < resultsLinesCount)
135+
{
136+
resultsStringBuilder.AppendLine();
137+
}
138+
}
139+
compositeInProcessDiagnoser.DeserializeResults(diagnoserIndex, benchmarkCase, resultsStringBuilder.ToString());
140+
}
141+
}
105142

106143
return new ExecuteResult(true,
107144
process.HasExited ? process.ExitCode : null,
108145
process.Id,
109-
outputLines.Where(line => !line.StartsWith("//")).ToArray(),
110-
outputLines.Where(line => line.StartsWith("//")).ToArray(),
146+
[.. prefixedLines],
147+
[.. standardOutput],
111148
outputLines,
112149
launchIndex);
113150
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
using BenchmarkDotNet.Analysers;
2+
using BenchmarkDotNet.Diagnosers;
3+
using BenchmarkDotNet.Engines;
4+
using BenchmarkDotNet.Exporters;
5+
using BenchmarkDotNet.Loggers;
6+
using BenchmarkDotNet.Reports;
7+
using BenchmarkDotNet.Running;
8+
using BenchmarkDotNet.Validators;
9+
using BenchmarkDotNet.Extensions;
10+
using System.Collections.Generic;
11+
using BenchmarkDotNet.Helpers;
12+
13+
namespace BenchmarkDotNet.IntegrationTests.Diagnosers
14+
{
15+
public sealed class MockInProcessDiagnoser : IInProcessDiagnoser
16+
{
17+
public Dictionary<BenchmarkCase, string> Results { get; } = [];
18+
19+
public IEnumerable<string> Ids => [nameof(MockInProcessDiagnoser)];
20+
21+
public IEnumerable<IExporter> Exporters => [];
22+
23+
public IEnumerable<IAnalyser> Analysers => [];
24+
25+
public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results);
26+
27+
public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoser)} results: [{string.Join(", ", Results.Values)}]");
28+
29+
public IInProcessDiagnoserHandler GetHandler(BenchmarkCase benchmarkCase, int index) => new MockInProcessDiagnoserHandler(index, GetRunMode(benchmarkCase));
30+
31+
public string GetHandlerSourceCode(BenchmarkCase benchmarkCase, int index)
32+
=> $"new {typeof(MockInProcessDiagnoserHandler).GetCorrectCSharpTypeName()}({index}, {SourceCodeHelper.ToSourceCode(GetRunMode(benchmarkCase))})";
33+
34+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
35+
36+
public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
37+
38+
public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
39+
40+
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
41+
}
42+
43+
public sealed class MockInProcessDiagnoserHandler(int index, RunMode runMode) : IInProcessDiagnoserHandler
44+
{
45+
public int Index { get; } = index;
46+
47+
public RunMode RunMode { get; } = runMode;
48+
49+
public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs parameters) { }
50+
51+
public string SerializeResults() => $"DummyResult{Index}";
52+
}
53+
}

tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using BenchmarkDotNet.Columns;
77
using BenchmarkDotNet.Configs;
88
using BenchmarkDotNet.Engines;
9+
using BenchmarkDotNet.IntegrationTests.Diagnosers;
910
using BenchmarkDotNet.IntegrationTests.InProcess.EmitTests;
1011
using BenchmarkDotNet.Jobs;
1112
using BenchmarkDotNet.Loggers;
@@ -130,6 +131,19 @@ public void InProcessBenchmarkEmitsSameIL(Type benchmarkType)
130131
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
131132
}
132133

134+
[Fact]
135+
public void InProcessEmitSupportsInProcessDiagnosers()
136+
{
137+
var logger = new OutputLogger(Output);
138+
var diagnoser = new MockInProcessDiagnoser();
139+
var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser);
140+
141+
var summary = CanExecute<BenchmarkAllCases>(config);
142+
143+
var expected = Enumerable.Repeat("DummyResult0", summary.BenchmarksCases.Length);
144+
Assert.Equal(expected, diagnoser.Results.Values);
145+
}
146+
133147
[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
134148
public class BenchmarkAllCases
135149
{

tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using BenchmarkDotNet.Configs;
1010
using BenchmarkDotNet.Engines;
1111
using BenchmarkDotNet.Environments;
12+
using BenchmarkDotNet.IntegrationTests.Diagnosers;
1213
using BenchmarkDotNet.Jobs;
1314
using BenchmarkDotNet.Loggers;
1415
using BenchmarkDotNet.Running;
@@ -217,6 +218,19 @@ public void InProcessBenchmarkAllCasesSupported()
217218
}
218219
}
219220

221+
[Fact]
222+
public void InProcessNoEmitSupportsInProcessDiagnosers()
223+
{
224+
var logger = new OutputLogger(Output);
225+
var diagnoser = new MockInProcessDiagnoser();
226+
var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser);
227+
228+
var summary = CanExecute<BenchmarkAllCases>(config);
229+
230+
var expected = Enumerable.Repeat("DummyResult0", summary.BenchmarksCases.Length);
231+
Assert.Equal(expected, diagnoser.Results.Values);
232+
}
233+
220234
[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
221235
public class BenchmarkAllCases
222236
{

0 commit comments

Comments
 (0)