diff --git a/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs b/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs new file mode 100644 index 0000000000..03d55574ec --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs @@ -0,0 +1,28 @@ +using BenchmarkDotNet.Attributes; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BenchmarkDotNet.Samples +{ + public class IntroMemoryRandomization + { + [Params(512 * 4)] + public int Size; + + private int[] _array; + private int[] _destination; + + [GlobalSetup] + public void Setup() + { + _array = new int[Size]; + _destination = new int[Size]; + } + + [Benchmark] + public void Array() => System.Array.Copy(_array, _destination, Size); + } +} diff --git a/src/BenchmarkDotNet/Attributes/Mutators/MemoryRandomizationAttribute.cs b/src/BenchmarkDotNet/Attributes/Mutators/MemoryRandomizationAttribute.cs new file mode 100644 index 0000000000..c921e2768e --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Mutators/MemoryRandomizationAttribute.cs @@ -0,0 +1,17 @@ +using BenchmarkDotNet.Jobs; +using Perfolizer.Mathematics.OutlierDetection; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// specifies whether Engine should allocate some random-sized memory between iterations + /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration + /// + public class MemoryRandomizationAttribute : JobMutatorConfigBaseAttribute + { + public MemoryRandomizationAttribute(bool enable = true, OutlierMode outlierMode = OutlierMode.DontRemove) + : base(Job.Default.WithMemoryRandomization(enable).WithOutlierMode(outlierMode)) + { + } + } +} diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index da31efd3f1..ab06e80363 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -168,6 +168,9 @@ public class CommandLineOptions [Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")] public IEnumerable EnvironmentVariables { get; set; } + [Option("memoryRandomization", Required = false, HelpText = "Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration.")] + public bool MemoryRandomization { get; set; } + [Option("wasmEngine", Required = false, HelpText = "Full path to a java script engine used to run the benchmarks, used by Wasm toolchain.")] public FileInfo WasmJavascriptEngine { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 04921600ca..0a09d2e65f 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -281,6 +281,8 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig) baseJob = baseJob.WithPlatform(options.Platform.Value); if (options.RunOncePerIteration) baseJob = baseJob.RunOncePerIteration(); + if (options.MemoryRandomization) + baseJob = baseJob.WithMemoryRandomization(); if (options.EnvironmentVariables.Any()) { diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 7ba21a4902..95672c0585 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -33,16 +35,18 @@ public class Engine : IEngine [PublicAPI] public string BenchmarkName { get; } private IClock Clock { get; } - private bool ForceAllocations { get; } + private bool ForceGcCleanups { get; } private int UnrollFactor { get; } private RunStrategy Strategy { get; } private bool EvaluateOverhead { get; } private int InvocationCount { get; } + private bool MemoryRandomization { get; } private readonly EnginePilotStage pilotStage; private readonly EngineWarmupStage warmupStage; private readonly EngineActualStage actualStage; private readonly bool includeExtraStats; + private readonly Random random; internal Engine( IHost host, @@ -70,15 +74,18 @@ internal Engine( Resolver = resolver; Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver); - ForceAllocations = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver); + ForceGcCleanups = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver); UnrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, Resolver); Strategy = targetJob.ResolveValue(RunMode.RunStrategyCharacteristic, Resolver); EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver); InvocationCount = targetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver); + MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver); warmupStage = new EngineWarmupStage(this); pilotStage = new EnginePilotStage(this); actualStage = new EngineActualStage(this); + + random = new Random(12345); // we are using constant seed to try to get repeatable results } public void Dispose() @@ -146,6 +153,7 @@ public Measurement RunIteration(IterationData data) int unrollFactor = data.UnrollFactor; long totalOperations = invokeCount * OperationsPerInvoke; bool isOverhead = data.IterationMode == IterationMode.Overhead; + bool randomizeMemory = !isOverhead && MemoryRandomization; var action = isOverhead ? OverheadAction : WorkloadAction; if (!isOverhead) @@ -156,6 +164,8 @@ public Measurement RunIteration(IterationData data) if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations); + Span stackMemory = randomizeMemory ? stackalloc byte[random.Next(32)] : Span.Empty; + // Measure var clock = Clock.Start(); action(invokeCount / unrollFactor); @@ -167,12 +177,17 @@ public Measurement RunIteration(IterationData data) if (!isOverhead) IterationCleanupAction(); + if (randomizeMemory) + RandomizeManagedHeapMemory(); + GcCollect(); // Results var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds()); WriteLine(measurement.ToString()); + Consume(stackMemory); + return measurement; } @@ -201,9 +216,30 @@ public Measurement RunIteration(IterationData data) return (gcStats, threadingStats); } + [MethodImpl(MethodImplOptions.NoInlining)] + private void Consume(in Span _) { } + + private void RandomizeManagedHeapMemory() + { + // invoke global cleanup before global setup + GlobalCleanupAction?.Invoke(); + + var gen0object = new byte[random.Next(32)]; + var lohObject = new byte[85 * 1024 + random.Next(32)]; + + // we expect the key allocations to happen in global setup (not ctor) + // so we call it while keeping the random-size objects alive + GlobalSetupAction?.Invoke(); + + GC.KeepAlive(gen0object); + GC.KeepAlive(lohObject); + + // we don't enforce GC.Collects here as engine does it later anyway + } + private void GcCollect() { - if (!ForceAllocations) + if (!ForceGcCleanups) return; ForceGcCollect(); diff --git a/src/BenchmarkDotNet/Engines/EngineResolver.cs b/src/BenchmarkDotNet/Engines/EngineResolver.cs index 8ecee9e93c..3fb93b84e0 100644 --- a/src/BenchmarkDotNet/Engines/EngineResolver.cs +++ b/src/BenchmarkDotNet/Engines/EngineResolver.cs @@ -33,8 +33,15 @@ private EngineResolver() Register(AccuracyMode.MinIterationTimeCharacteristic, () => TimeInterval.Millisecond * 500); Register(AccuracyMode.MinInvokeCountCharacteristic, () => 4); Register(AccuracyMode.EvaluateOverheadCharacteristic, () => true); + Register(RunMode.MemoryRandomizationCharacteristic, () => false); Register(AccuracyMode.OutlierModeCharacteristic, job => { + // if Memory Randomization was enabled and the benchmark is truly multimodal + // removing outliers could remove some values that are not actually outliers + // see https://github.com/dotnet/BenchmarkDotNet/pull/1587#issue-516837573 for example + if (job.ResolveValue(RunMode.MemoryRandomizationCharacteristic, this)) + return OutlierMode.DontRemove; + var strategy = job.ResolveValue(RunMode.RunStrategyCharacteristic, this); switch (strategy) { diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 165c648195..335e8efb8d 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -210,12 +210,17 @@ public static Job WithHeapAffinitizeMask(this Job job, int heapAffinitizeMask) = /// public static Job WithPowerPlan(this Job job, Guid powerPlanGuid) => job.WithCore(j => j.Environment.PowerPlanMode = powerPlanGuid); - /// /// ensures that BenchmarkDotNet does not enforce any power plan /// public static Job DontEnforcePowerPlan(this Job job) => job.WithCore(j => j.Environment.PowerPlanMode = Guid.Empty); + /// + /// specifies whether Engine should allocate some random-sized memory between iterations + /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration + /// + public static Job WithMemoryRandomization(this Job job, bool enable = true) => job.WithCore(j => j.Run.MemoryRandomization = enable); + // Infrastructure [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This method will soon be removed, please start using .WithToolchain instead")] diff --git a/src/BenchmarkDotNet/Jobs/RunMode.cs b/src/BenchmarkDotNet/Jobs/RunMode.cs index 2ca534678b..d3248b0921 100644 --- a/src/BenchmarkDotNet/Jobs/RunMode.cs +++ b/src/BenchmarkDotNet/Jobs/RunMode.cs @@ -21,6 +21,7 @@ public sealed class RunMode : JobMode public static readonly Characteristic WarmupCountCharacteristic = CreateCharacteristic(nameof(WarmupCount)); public static readonly Characteristic MinWarmupIterationCountCharacteristic = CreateCharacteristic(nameof(MinWarmupIterationCount)); public static readonly Characteristic MaxWarmupIterationCountCharacteristic = CreateCharacteristic(nameof(MaxWarmupIterationCount)); + public static readonly Characteristic MemoryRandomizationCharacteristic = CreateCharacteristic(nameof(MemoryRandomization)); public static readonly RunMode Dry = new RunMode(nameof(Dry)) { @@ -180,5 +181,15 @@ public int MaxWarmupIterationCount get { return MaxWarmupIterationCountCharacteristic[this]; } set { MaxWarmupIterationCountCharacteristic[this] = value; } } + + /// + /// specifies whether Engine should allocate some random-sized memory between iterations + /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration + /// + public bool MemoryRandomization + { + get => MemoryRandomizationCharacteristic[this]; + set => MemoryRandomizationCharacteristic[this] = value; + } } } \ No newline at end of file