diff --git a/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs b/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs
new file mode 100644
index 0000000000..03d55574ec
--- /dev/null
+++ b/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs
@@ -0,0 +1,28 @@
+using BenchmarkDotNet.Attributes;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BenchmarkDotNet.Samples
+{
+ public class IntroMemoryRandomization
+ {
+ [Params(512 * 4)]
+ public int Size;
+
+ private int[] _array;
+ private int[] _destination;
+
+ [GlobalSetup]
+ public void Setup()
+ {
+ _array = new int[Size];
+ _destination = new int[Size];
+ }
+
+ [Benchmark]
+ public void Array() => System.Array.Copy(_array, _destination, Size);
+ }
+}
diff --git a/src/BenchmarkDotNet/Attributes/Mutators/MemoryRandomizationAttribute.cs b/src/BenchmarkDotNet/Attributes/Mutators/MemoryRandomizationAttribute.cs
new file mode 100644
index 0000000000..c921e2768e
--- /dev/null
+++ b/src/BenchmarkDotNet/Attributes/Mutators/MemoryRandomizationAttribute.cs
@@ -0,0 +1,17 @@
+using BenchmarkDotNet.Jobs;
+using Perfolizer.Mathematics.OutlierDetection;
+
+namespace BenchmarkDotNet.Attributes
+{
+ ///
+ /// specifies whether Engine should allocate some random-sized memory between iterations
+ /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration
+ ///
+ public class MemoryRandomizationAttribute : JobMutatorConfigBaseAttribute
+ {
+ public MemoryRandomizationAttribute(bool enable = true, OutlierMode outlierMode = OutlierMode.DontRemove)
+ : base(Job.Default.WithMemoryRandomization(enable).WithOutlierMode(outlierMode))
+ {
+ }
+ }
+}
diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs
index da31efd3f1..ab06e80363 100644
--- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs
+++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs
@@ -168,6 +168,9 @@ public class CommandLineOptions
[Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")]
public IEnumerable EnvironmentVariables { get; set; }
+ [Option("memoryRandomization", Required = false, HelpText = "Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration.")]
+ public bool MemoryRandomization { get; set; }
+
[Option("wasmEngine", Required = false, HelpText = "Full path to a java script engine used to run the benchmarks, used by Wasm toolchain.")]
public FileInfo WasmJavascriptEngine { get; set; }
diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs
index 04921600ca..0a09d2e65f 100644
--- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs
+++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs
@@ -281,6 +281,8 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig)
baseJob = baseJob.WithPlatform(options.Platform.Value);
if (options.RunOncePerIteration)
baseJob = baseJob.RunOncePerIteration();
+ if (options.MemoryRandomization)
+ baseJob = baseJob.WithMemoryRandomization();
if (options.EnvironmentVariables.Any())
{
diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs
index 7ba21a4902..95672c0585 100644
--- a/src/BenchmarkDotNet/Engines/Engine.cs
+++ b/src/BenchmarkDotNet/Engines/Engine.cs
@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics;
using System.Globalization;
using System.Linq;
+using System.Runtime.CompilerServices;
using BenchmarkDotNet.Characteristics;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Portability;
@@ -33,16 +35,18 @@ public class Engine : IEngine
[PublicAPI] public string BenchmarkName { get; }
private IClock Clock { get; }
- private bool ForceAllocations { get; }
+ private bool ForceGcCleanups { get; }
private int UnrollFactor { get; }
private RunStrategy Strategy { get; }
private bool EvaluateOverhead { get; }
private int InvocationCount { get; }
+ private bool MemoryRandomization { get; }
private readonly EnginePilotStage pilotStage;
private readonly EngineWarmupStage warmupStage;
private readonly EngineActualStage actualStage;
private readonly bool includeExtraStats;
+ private readonly Random random;
internal Engine(
IHost host,
@@ -70,15 +74,18 @@ internal Engine(
Resolver = resolver;
Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver);
- ForceAllocations = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver);
+ ForceGcCleanups = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver);
UnrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, Resolver);
Strategy = targetJob.ResolveValue(RunMode.RunStrategyCharacteristic, Resolver);
EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver);
InvocationCount = targetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver);
+ MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver);
warmupStage = new EngineWarmupStage(this);
pilotStage = new EnginePilotStage(this);
actualStage = new EngineActualStage(this);
+
+ random = new Random(12345); // we are using constant seed to try to get repeatable results
}
public void Dispose()
@@ -146,6 +153,7 @@ public Measurement RunIteration(IterationData data)
int unrollFactor = data.UnrollFactor;
long totalOperations = invokeCount * OperationsPerInvoke;
bool isOverhead = data.IterationMode == IterationMode.Overhead;
+ bool randomizeMemory = !isOverhead && MemoryRandomization;
var action = isOverhead ? OverheadAction : WorkloadAction;
if (!isOverhead)
@@ -156,6 +164,8 @@ public Measurement RunIteration(IterationData data)
if (EngineEventSource.Log.IsEnabled())
EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations);
+ Span stackMemory = randomizeMemory ? stackalloc byte[random.Next(32)] : Span.Empty;
+
// Measure
var clock = Clock.Start();
action(invokeCount / unrollFactor);
@@ -167,12 +177,17 @@ public Measurement RunIteration(IterationData data)
if (!isOverhead)
IterationCleanupAction();
+ if (randomizeMemory)
+ RandomizeManagedHeapMemory();
+
GcCollect();
// Results
var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds());
WriteLine(measurement.ToString());
+ Consume(stackMemory);
+
return measurement;
}
@@ -201,9 +216,30 @@ public Measurement RunIteration(IterationData data)
return (gcStats, threadingStats);
}
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private void Consume(in Span _) { }
+
+ private void RandomizeManagedHeapMemory()
+ {
+ // invoke global cleanup before global setup
+ GlobalCleanupAction?.Invoke();
+
+ var gen0object = new byte[random.Next(32)];
+ var lohObject = new byte[85 * 1024 + random.Next(32)];
+
+ // we expect the key allocations to happen in global setup (not ctor)
+ // so we call it while keeping the random-size objects alive
+ GlobalSetupAction?.Invoke();
+
+ GC.KeepAlive(gen0object);
+ GC.KeepAlive(lohObject);
+
+ // we don't enforce GC.Collects here as engine does it later anyway
+ }
+
private void GcCollect()
{
- if (!ForceAllocations)
+ if (!ForceGcCleanups)
return;
ForceGcCollect();
diff --git a/src/BenchmarkDotNet/Engines/EngineResolver.cs b/src/BenchmarkDotNet/Engines/EngineResolver.cs
index 8ecee9e93c..3fb93b84e0 100644
--- a/src/BenchmarkDotNet/Engines/EngineResolver.cs
+++ b/src/BenchmarkDotNet/Engines/EngineResolver.cs
@@ -33,8 +33,15 @@ private EngineResolver()
Register(AccuracyMode.MinIterationTimeCharacteristic, () => TimeInterval.Millisecond * 500);
Register(AccuracyMode.MinInvokeCountCharacteristic, () => 4);
Register(AccuracyMode.EvaluateOverheadCharacteristic, () => true);
+ Register(RunMode.MemoryRandomizationCharacteristic, () => false);
Register(AccuracyMode.OutlierModeCharacteristic, job =>
{
+ // if Memory Randomization was enabled and the benchmark is truly multimodal
+ // removing outliers could remove some values that are not actually outliers
+ // see https://github.com/dotnet/BenchmarkDotNet/pull/1587#issue-516837573 for example
+ if (job.ResolveValue(RunMode.MemoryRandomizationCharacteristic, this))
+ return OutlierMode.DontRemove;
+
var strategy = job.ResolveValue(RunMode.RunStrategyCharacteristic, this);
switch (strategy)
{
diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs
index 165c648195..335e8efb8d 100644
--- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs
+++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs
@@ -210,12 +210,17 @@ public static Job WithHeapAffinitizeMask(this Job job, int heapAffinitizeMask) =
///
public static Job WithPowerPlan(this Job job, Guid powerPlanGuid) => job.WithCore(j => j.Environment.PowerPlanMode = powerPlanGuid);
-
///
/// ensures that BenchmarkDotNet does not enforce any power plan
///
public static Job DontEnforcePowerPlan(this Job job) => job.WithCore(j => j.Environment.PowerPlanMode = Guid.Empty);
+ ///
+ /// specifies whether Engine should allocate some random-sized memory between iterations
+ /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration
+ ///
+ public static Job WithMemoryRandomization(this Job job, bool enable = true) => job.WithCore(j => j.Run.MemoryRandomization = enable);
+
// Infrastructure
[EditorBrowsable(EditorBrowsableState.Never)]
[Obsolete("This method will soon be removed, please start using .WithToolchain instead")]
diff --git a/src/BenchmarkDotNet/Jobs/RunMode.cs b/src/BenchmarkDotNet/Jobs/RunMode.cs
index 2ca534678b..d3248b0921 100644
--- a/src/BenchmarkDotNet/Jobs/RunMode.cs
+++ b/src/BenchmarkDotNet/Jobs/RunMode.cs
@@ -21,6 +21,7 @@ public sealed class RunMode : JobMode
public static readonly Characteristic WarmupCountCharacteristic = CreateCharacteristic(nameof(WarmupCount));
public static readonly Characteristic MinWarmupIterationCountCharacteristic = CreateCharacteristic(nameof(MinWarmupIterationCount));
public static readonly Characteristic MaxWarmupIterationCountCharacteristic = CreateCharacteristic(nameof(MaxWarmupIterationCount));
+ public static readonly Characteristic MemoryRandomizationCharacteristic = CreateCharacteristic(nameof(MemoryRandomization));
public static readonly RunMode Dry = new RunMode(nameof(Dry))
{
@@ -180,5 +181,15 @@ public int MaxWarmupIterationCount
get { return MaxWarmupIterationCountCharacteristic[this]; }
set { MaxWarmupIterationCountCharacteristic[this] = value; }
}
+
+ ///
+ /// specifies whether Engine should allocate some random-sized memory between iterations
+ /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration
+ ///
+ public bool MemoryRandomization
+ {
+ get => MemoryRandomizationCharacteristic[this];
+ set => MemoryRandomizationCharacteristic[this] = value;
+ }
}
}
\ No newline at end of file