Skip to content

Commit d7ceb27

Browse files
timcasselltimcassell
committed
Fix async GlobalSetup/GlobalCleanup not being awaited with InProcessEmit toolchain.
1 parent 5720494 commit d7ceb27

File tree

4 files changed

+276
-28
lines changed

4 files changed

+276
-28
lines changed

‎src/BenchmarkDotNet/Toolchains/InProcess.Emit.Implementation/Emitters/RunnableEmitter.cs‎

Lines changed: 82 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,10 @@ private static void EmitNoArgsMethodCallPopReturn(
245245
private TypeBuilder runnableBuilder;
246246
private ConsumableTypeInfo consumableInfo;
247247
private ConsumeEmitter consumeEmitter;
248+
private ConsumableTypeInfo globalSetupReturnInfo;
249+
private ConsumableTypeInfo globalCleanupReturnInfo;
250+
private ConsumableTypeInfo iterationSetupReturnInfo;
251+
private ConsumableTypeInfo iterationCleanupReturnInfo;
248252

249253
private FieldBuilder awaitHelperField;
250254
private FieldBuilder globalSetupActionField;
@@ -358,13 +362,22 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark)
358362

359363
consumableInfo = new ConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.WorkloadMethod.ReturnType);
360364
consumeEmitter = ConsumeEmitter.GetConsumeEmitter(consumableInfo);
365+
globalSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalSetupMethod?.ReturnType);
366+
globalCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalCleanupMethod?.ReturnType);
367+
iterationSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationSetupMethod?.ReturnType);
368+
iterationCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationCleanupMethod?.ReturnType);
361369

362370
// Init types
363371
runnableBuilder = DefineRunnableTypeBuilder(benchmark, moduleBuilder);
364372
overheadDelegateType = EmitOverheadDelegateType();
365373
workloadDelegateType = EmitWorkloadDelegateType();
366374
}
367375

376+
private static ConsumableTypeInfo GetConsumableTypeInfo(Type methodReturnType)
377+
{
378+
return methodReturnType == null ? null : new ConsumableTypeInfo(methodReturnType);
379+
}
380+
368381
private Type EmitOverheadDelegateType()
369382
{
370383
// .class public auto ansi sealed BenchmarkDotNet.Autogenerated.Runnable_0OverheadDelegate
@@ -908,34 +921,89 @@ private void EmitSetupCleanupMethods()
908921
{
909922
// Emit Setup/Cleanup methods
910923
// We emit empty method instead of EmptyAction = "() => { }"
911-
globalSetupMethod = EmitWrapperMethod(
912-
GlobalSetupMethodName,
913-
Descriptor.GlobalSetupMethod);
914-
globalCleanupMethod = EmitWrapperMethod(
915-
GlobalCleanupMethodName,
916-
Descriptor.GlobalCleanupMethod);
917-
iterationSetupMethod = EmitWrapperMethod(
918-
IterationSetupMethodName,
919-
Descriptor.IterationSetupMethod);
920-
iterationCleanupMethod = EmitWrapperMethod(
921-
IterationCleanupMethodName,
922-
Descriptor.IterationCleanupMethod);
924+
globalSetupMethod = EmitWrapperMethod(GlobalSetupMethodName, Descriptor.GlobalSetupMethod, globalSetupReturnInfo);
925+
globalCleanupMethod = EmitWrapperMethod(GlobalCleanupMethodName, Descriptor.GlobalCleanupMethod, globalCleanupReturnInfo);
926+
iterationSetupMethod = EmitWrapperMethod(IterationSetupMethodName, Descriptor.IterationSetupMethod, iterationSetupReturnInfo);
927+
iterationCleanupMethod = EmitWrapperMethod(IterationCleanupMethodName, Descriptor.IterationCleanupMethod, iterationCleanupReturnInfo);
923928
}
924929

925-
private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod)
930+
private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod, ConsumableTypeInfo returnTypeInfo)
926931
{
927932
var methodBuilder = runnableBuilder.DefinePrivateVoidInstanceMethod(methodName);
928933

929934
var ilBuilder = methodBuilder.GetILGenerator();
930935

931936
if (optionalTargetMethod != null)
932-
EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true);
937+
{
938+
if (returnTypeInfo?.IsAwaitable == true)
939+
{
940+
EmitAwaitableSetupTeardown(methodBuilder, optionalTargetMethod, ilBuilder, returnTypeInfo);
941+
}
942+
else
943+
{
944+
EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true);
945+
}
946+
}
933947

934948
ilBuilder.EmitVoidReturn(methodBuilder);
935949

936950
return methodBuilder;
937951
}
938952

953+
private void EmitAwaitableSetupTeardown(
954+
MethodBuilder methodBuilder,
955+
MethodInfo targetMethod,
956+
ILGenerator ilBuilder,
957+
ConsumableTypeInfo returnTypeInfo)
958+
{
959+
if (targetMethod == null)
960+
throw new ArgumentNullException(nameof(targetMethod));
961+
962+
if (returnTypeInfo.WorkloadMethodReturnType == typeof(void))
963+
{
964+
ilBuilder.Emit(OpCodes.Ldarg_0);
965+
}
966+
/*
967+
IL_0000: ldarg.0
968+
IL_0001: ldfld class BenchmarkDotNet.Helpers.AwaitHelper BenchmarkDotNet.Helpers.Runnable_0::awaitHelper
969+
*/
970+
ilBuilder.Emit(OpCodes.Ldarg_0);
971+
ilBuilder.Emit(OpCodes.Ldfld, awaitHelperField);
972+
/*
973+
// call for instance
974+
// GlobalSetup();
975+
IL_0006: ldarg.0
976+
IL_0007: call instance void [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalSetup()
977+
*/
978+
/*
979+
// call for static
980+
// GlobalSetup();
981+
IL_0006: call string [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalCleanup()
982+
*/
983+
if (targetMethod.IsStatic)
984+
{
985+
ilBuilder.Emit(OpCodes.Call, targetMethod);
986+
987+
}
988+
else if (methodBuilder.IsStatic)
989+
{
990+
throw new InvalidOperationException(
991+
$"[BUG] Static method {methodBuilder.Name} tries to call instance member {targetMethod.Name}");
992+
}
993+
else
994+
{
995+
ilBuilder.Emit(OpCodes.Ldarg_0);
996+
ilBuilder.Emit(OpCodes.Call, targetMethod);
997+
}
998+
999+
/*
1000+
// awaitHelper.GetResult(...);
1001+
IL_000e: callvirt instance void BenchmarkDotNet.Helpers.AwaitHelper::GetResult(class [System.Private.CoreLib]System.Threading.Tasks.Task)
1002+
*/
1003+
ilBuilder.Emit(OpCodes.Callvirt, returnTypeInfo.GetResultMethod);
1004+
ilBuilder.Emit(OpCodes.Pop);
1005+
}
1006+
9391007
private void EmitCtorBody()
9401008
{
9411009
var ilBuilder = ctorMethod.GetILGenerator();

‎tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,5 +258,97 @@ public ValueTask<int> GlobalCleanup()
258258
[Benchmark]
259259
public void Benchmark() => Console.WriteLine(BenchmarkCalled);
260260
}
261+
262+
[Fact]
263+
public void AllSetupAndCleanupMethodRunsAsyncValueTaskBackByIValueTaskSourceSetupTest()
264+
{
265+
var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob");
266+
var config = CreateSimpleConfig(job: miniJob);
267+
268+
var summary = CanExecute<AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskSetup>(config);
269+
270+
var actualLogLines = GetActualLogLines(summary);
271+
foreach (string line in actualLogLines)
272+
Output.WriteLine(line);
273+
Assert.Equal(expectedLogLines, actualLogLines);
274+
}
275+
276+
public class AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskBackByIValueTaskSourceSetup
277+
{
278+
private readonly ValueTaskSource<int> valueTaskSource = new ();
279+
private int setupCounter;
280+
private int cleanupCounter;
281+
282+
[IterationSetup]
283+
public void IterationSetup() => Console.WriteLine(IterationSetupCalled + " (" + ++setupCounter + ")");
284+
285+
[IterationCleanup]
286+
public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")");
287+
288+
[GlobalSetup]
289+
public ValueTask GlobalSetup()
290+
{
291+
valueTaskSource.Reset();
292+
Console.Out.WriteLineAsync(GlobalSetupCalled).ContinueWith(_ => valueTaskSource.SetResult(42));
293+
return new ValueTask(valueTaskSource, valueTaskSource.Token);
294+
}
295+
296+
[GlobalCleanup]
297+
public ValueTask GlobalCleanup()
298+
{
299+
valueTaskSource.Reset();
300+
Console.Out.WriteLineAsync(GlobalCleanupCalled).ContinueWith(_ => valueTaskSource.SetResult(42));
301+
return new ValueTask(valueTaskSource, valueTaskSource.Token);
302+
}
303+
304+
[Benchmark]
305+
public void Benchmark() => Console.WriteLine(BenchmarkCalled);
306+
}
307+
308+
[FactNotGitHubActionsWindows]
309+
public void AllSetupAndCleanupMethodRunsAsyncGenericValueTaskBackByIValueTaskSourceSetupTest()
310+
{
311+
var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob");
312+
var config = CreateSimpleConfig(job: miniJob);
313+
314+
var summary = CanExecute<AllSetupAndCleanupAttributeBenchmarksAsyncGenericValueTaskSetup>(config);
315+
316+
var actualLogLines = GetActualLogLines(summary);
317+
foreach (string line in actualLogLines)
318+
Output.WriteLine(line);
319+
Assert.Equal(expectedLogLines, actualLogLines);
320+
}
321+
322+
public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericValueTaskBackByIValueTaskSourceSetup
323+
{
324+
private readonly ValueTaskSource<int> valueTaskSource = new ();
325+
private int setupCounter;
326+
private int cleanupCounter;
327+
328+
[IterationSetup]
329+
public void IterationSetup() => Console.WriteLine(IterationSetupCalled + " (" + ++setupCounter + ")");
330+
331+
[IterationCleanup]
332+
public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")");
333+
334+
[GlobalSetup]
335+
public ValueTask<int> GlobalSetup()
336+
{
337+
valueTaskSource.Reset();
338+
Console.Out.WriteLineAsync(GlobalSetupCalled).ContinueWith(_ => valueTaskSource.SetResult(42));
339+
return new ValueTask<int>(valueTaskSource, valueTaskSource.Token);
340+
}
341+
342+
[GlobalCleanup]
343+
public ValueTask<int> GlobalCleanup()
344+
{
345+
valueTaskSource.Reset();
346+
Console.Out.WriteLineAsync(GlobalCleanupCalled).ContinueWith(_ => valueTaskSource.SetResult(42));
347+
return new ValueTask<int>(valueTaskSource, valueTaskSource.Token);
348+
}
349+
350+
[Benchmark]
351+
public void Benchmark() => Console.WriteLine(BenchmarkCalled);
352+
}
261353
}
262354
}

‎tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public Reports.Summary CanExecute<TBenchmark>(IConfig config = null, bool fullVa
4949
/// <param name="config">Optional custom config to be used instead of the default</param>
5050
/// <param name="fullValidation">Optional: disable validation (default = true/enabled)</param>
5151
/// <returns>The summary from the benchmark run</returns>
52-
protected Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true)
52+
public Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true)
5353
{
5454
// Add logging, so the Benchmark execution is in the TestRunner output (makes Debugging easier)
5555
if (config == null)

‎tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs‎

Lines changed: 101 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -235,35 +235,123 @@ public static ValueTask<decimal> InvokeOnceStaticValueTaskOfT()
235235
}
236236
}
237237

238-
[Fact]
239-
public void InProcessEmitToolchainSupportsIterationSetupAndCleanup()
238+
[Theory]
239+
[InlineData(typeof(IterationSetupCleanup))]
240+
[InlineData(typeof(GlobalSetupCleanupTask))]
241+
[InlineData(typeof(GlobalSetupCleanupValueTask))]
242+
[InlineData(typeof(GlobalSetupCleanupValueTaskSource))]
243+
public void InProcessEmitToolchainSupportsSetupAndCleanup(Type benchmarkType)
240244
{
241245
var logger = new OutputLogger(Output);
242246
var config = CreateInProcessConfig(logger);
243247

244-
WithIterationSetupAndCleanup.SetupCounter = 0;
245-
WithIterationSetupAndCleanup.BenchmarkCounter = 0;
246-
WithIterationSetupAndCleanup.CleanupCounter = 0;
248+
Counters.SetupCounter = 0;
249+
Counters.BenchmarkCounter = 0;
250+
Counters.CleanupCounter = 0;
247251

248-
var summary = CanExecute<WithIterationSetupAndCleanup>(config);
252+
var summary = CanExecute(benchmarkType, config);
249253

250-
Assert.Equal(1, WithIterationSetupAndCleanup.SetupCounter);
251-
Assert.Equal(16, WithIterationSetupAndCleanup.BenchmarkCounter);
252-
Assert.Equal(1, WithIterationSetupAndCleanup.CleanupCounter);
254+
Assert.Equal(1, Counters.SetupCounter);
255+
Assert.Equal(16, Counters.BenchmarkCounter);
256+
Assert.Equal(1, Counters.CleanupCounter);
253257
}
254258

255-
public class WithIterationSetupAndCleanup
259+
private static class Counters
256260
{
257261
public static int SetupCounter, BenchmarkCounter, CleanupCounter;
262+
}
258263

264+
public class IterationSetupCleanup
265+
{
259266
[IterationSetup]
260-
public void Setup() => Interlocked.Increment(ref SetupCounter);
267+
public void Setup() => Interlocked.Increment(ref Counters.SetupCounter);
261268

262269
[Benchmark]
263-
public void Benchmark() => Interlocked.Increment(ref BenchmarkCounter);
270+
public void Benchmark() => Interlocked.Increment(ref Counters.BenchmarkCounter);
264271

265272
[IterationCleanup]
266-
public void Cleanup() => Interlocked.Increment(ref CleanupCounter);
273+
public void Cleanup() => Interlocked.Increment(ref Counters.CleanupCounter);
274+
}
275+
276+
public class GlobalSetupCleanupTask
277+
{
278+
[GlobalSetup]
279+
public static async Task GlobalSetup()
280+
{
281+
await Task.Yield();
282+
Interlocked.Increment(ref Counters.SetupCounter);
283+
}
284+
285+
[GlobalCleanup]
286+
public async Task GlobalCleanup()
287+
{
288+
await Task.Yield();
289+
Interlocked.Increment(ref Counters.CleanupCounter);
290+
}
291+
292+
[Benchmark]
293+
public void InvokeOnceVoid()
294+
{
295+
Interlocked.Increment(ref Counters.BenchmarkCounter);
296+
}
297+
}
298+
299+
public class GlobalSetupCleanupValueTask
300+
{
301+
[GlobalSetup]
302+
public static async ValueTask GlobalSetup()
303+
{
304+
await Task.Yield();
305+
Interlocked.Increment(ref Counters.SetupCounter);
306+
}
307+
308+
[GlobalCleanup]
309+
public async ValueTask GlobalCleanup()
310+
{
311+
await Task.Yield();
312+
Interlocked.Increment(ref Counters.CleanupCounter);
313+
}
314+
315+
[Benchmark]
316+
public void InvokeOnceVoid()
317+
{
318+
Interlocked.Increment(ref Counters.BenchmarkCounter);
319+
}
320+
}
321+
322+
public class GlobalSetupCleanupValueTaskSource
323+
{
324+
private readonly static ValueTaskSource<int> valueTaskSource = new ();
325+
326+
[GlobalSetup]
327+
public static ValueTask GlobalSetup()
328+
{
329+
valueTaskSource.Reset();
330+
Task.Delay(1).ContinueWith(_ =>
331+
{
332+
Interlocked.Increment(ref Counters.SetupCounter);
333+
valueTaskSource.SetResult(42);
334+
});
335+
return new ValueTask(valueTaskSource, valueTaskSource.Token);
336+
}
337+
338+
[GlobalCleanup]
339+
public ValueTask GlobalCleanup()
340+
{
341+
valueTaskSource.Reset();
342+
Task.Delay(1).ContinueWith(_ =>
343+
{
344+
Interlocked.Increment(ref Counters.CleanupCounter);
345+
valueTaskSource.SetResult(42);
346+
});
347+
return new ValueTask(valueTaskSource, valueTaskSource.Token);
348+
}
349+
350+
[Benchmark]
351+
public void InvokeOnceVoid()
352+
{
353+
Interlocked.Increment(ref Counters.BenchmarkCounter);
354+
}
267355
}
268356
}
269357
}

0 commit comments

Comments
 (0)