Added --memorySurvived command line option.

timcassell · timcassell · commit 410414347fad · 2020-12-04T05:12:42.000-05:00
Added WorkloadActionSingleInvoke.
Only measure survived memory for the first WorkloadAction, using the SingleInvoke, instead of every iteration.
diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs
@@ -47,6 +47,9 @@ public class CommandLineOptions
         [Option('a', "artifacts", Required = false, HelpText = "Valid path to accessible directory")]
         public DirectoryInfo ArtifactsDirectory { get; set; }
 
+        [Option("memorySurvived", Required = false, Default = false, HelpText = "Measures survived memory.")]
+        public bool UseSurvivedMemoryDiagnoser { get; set; }
+
         [Option("outliers", Required = false, Default = OutlierMode.RemoveUpper, HelpText = "DontRemove/RemoveUpper/RemoveLower/RemoveAll")]
         public OutlierMode Outliers { get; set; }
 
diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs
@@ -193,8 +193,11 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo
                 .Select(counterName => (HardwareCounter)Enum.Parse(typeof(HardwareCounter), counterName, ignoreCase: true))
                 .ToArray());
 
-            if (options.UseMemoryDiagnoser)
+            if (options.UseSurvivedMemoryDiagnoser)
+                config.AddDiagnoser(MemoryDiagnoser.WithSurvived);
+            else if (options.UseMemoryDiagnoser)
                 config.AddDiagnoser(MemoryDiagnoser.Default);
+
             if (options.UseThreadingDiagnoser)
                 config.AddDiagnoser(ThreadingDiagnoser.Default);
             if (options.UseDisassemblyDiagnoser)
diff --git a/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs
@@ -68,7 +68,7 @@ private class SurvivedMemoryMetricDescriptor : IMetricDescriptor
 
             public string Id => "Survived Memory";
             public string DisplayName => "Survived";
-            public string Legend => "Total memory survived after all operations (managed only, inclusive, 1KB = 1024B)";
+            public string Legend => "Memory survived after the first operation (managed only, inclusive, 1KB = 1024B)";
             public string NumberFormat => "N0";
             public UnitType UnitType => UnitType.Size;
             public string Unit => SizeUnit.B.Name;
diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs
@@ -19,6 +19,7 @@ public class Engine : IEngine
 
         [PublicAPI] public IHost Host { get; }
         [PublicAPI] public Action<long> WorkloadAction { get; }
+        [PublicAPI] public Action WorkloadActionSingleInvoke { get; }
         [PublicAPI] public Action Dummy1Action { get; }
         [PublicAPI] public Action Dummy2Action { get; }
         [PublicAPI] public Action Dummy3Action { get; }
@@ -45,18 +46,21 @@ public class Engine : IEngine
         private readonly EngineActualStage actualStage;
         private readonly bool includeExtraStats, includeSurvivedMemory;
 
-        private long _totalMeasuredSurvivedBytes;
+        // These must be static since more than one Engine is used.
+        private static long survivedBytes;
+        private static bool survivedBytesMeasured;
         private Func<long> GetTotalBytes { get; }
 
         internal Engine(
             IHost host,
             IResolver resolver,
-            Action dummy1Action, Action dummy2Action, Action dummy3Action, Action<long> overheadAction, Action<long> workloadAction, Job targetJob,
+            Action dummy1Action, Action dummy2Action, Action dummy3Action, Action<long> overheadAction, Action<long> workloadAction, Action workloadActionSingleInvoke, Job targetJob,
             Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke,
             bool includeExtraStats, bool includeSurvivedMemory, string benchmarkName)
         {
             Host = host;
             OverheadAction = overheadAction;
+            WorkloadActionSingleInvoke = workloadActionSingleInvoke;
             Dummy1Action = dummy1Action;
             Dummy2Action = dummy2Action;
             Dummy3Action = dummy3Action;
@@ -84,14 +88,15 @@ internal Engine(
             pilotStage = new EnginePilotStage(this);
             actualStage = new EngineActualStage(this);
 
-            GetTotalBytes = GetTotalBytesFunc();
-            // Necessary for CORE runtimes.
-            if (includeSurvivedMemory)
+            if (includeSurvivedMemory && !survivedBytesMeasured)
             {
+                GetTotalBytes = GetTotalBytesFunc();
+
+                // Necessary for CORE runtimes.
                 // Measure bytes to allow GC monitor to make its allocations.
                 GetTotalBytes();
                 // Run the clock once to allow it to make its allocations.
-                MeasureAction(_ => { }, 0);
+                MeasureAction(() => { });
                 GetTotalBytes();
             }
         }
@@ -100,10 +105,8 @@ private Func<long> GetTotalBytesFunc()
         {
             // Only enable monitoring if memory diagnoser with survived memory is applied.
             // Don't try to measure in Mono, Monitoring is not available, and GC.GetTotalMemory is very inaccurate.
-            if (!includeSurvivedMemory || RuntimeInformation.IsMono)
-            {
+            if (RuntimeInformation.IsMono)
                 return () => 0;
-            }
             try
             {
                 // Docs say this should be available in .NET Core 2.1, but it throws an exception.
@@ -194,20 +197,32 @@ public Measurement RunIteration(IterationData data)
             bool isOverhead = data.IterationMode == IterationMode.Overhead;
             var action = isOverhead ? OverheadAction : WorkloadAction;
 
+            double nanoseconds = 0;
             if (!isOverhead)
+            {
                 IterationSetupAction();
 
+                if (includeSurvivedMemory && !survivedBytesMeasured)
+                {
+                    // Measure survived bytes for only the first invocation.
+                    survivedBytesMeasured = true;
+                    ++totalOperations;
+                    long beforeBytes = GetTotalBytes();
+                    nanoseconds = MeasureAction(WorkloadActionSingleInvoke);
+                    long afterBytes = GetTotalBytes();
+                    survivedBytes = afterBytes - beforeBytes;
+                }
+            }
+
             GcCollect();
 
             if (EngineEventSource.Log.IsEnabled())
                 EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations);
 
             // Measure
-            long beforeBytes = GetTotalBytes();
-            double nanoseconds = MeasureAction(action, invokeCount / unrollFactor);
-            long afterBytes = GetTotalBytes();
-            long survivedBytes = afterBytes - beforeBytes;
-            _totalMeasuredSurvivedBytes += survivedBytes;
+            var clock = Clock.Start();
+            action(invokeCount / unrollFactor);
+            nanoseconds += clock.GetElapsed().GetNanoseconds();
 
             if (EngineEventSource.Log.IsEnabled())
                 EngineEventSource.Log.IterationStop(data.IterationMode, data.IterationStage, totalOperations);
@@ -218,18 +233,18 @@ public Measurement RunIteration(IterationData data)
             GcCollect();
 
             // Results
-            var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, nanoseconds, survivedBytes);
+            var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, nanoseconds);
             WriteLine(measurement.ToString());
 
             return measurement;
         }
 
         // This is necessary for the CORE runtime to clean up the memory from the clock.
         [MethodImpl(MethodImplOptions.NoInlining)]
-        private double MeasureAction(Action<long> action, long arg)
+        private double MeasureAction(Action action)
         {
             var clock = Clock.Start();
-            action(arg);
+            action();
             return clock.GetElapsed().GetNanoseconds();
         }
 
@@ -252,7 +267,7 @@ private double MeasureAction(Action<long> action, long arg)
 
             IterationCleanupAction(); // we run iteration cleanup after collecting GC stats
 
-            GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperationsAndSurvivedBytes(data.InvokeCount * OperationsPerInvoke, _totalMeasuredSurvivedBytes);
+            GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperationsAndSurvivedBytes(data.InvokeCount * OperationsPerInvoke, survivedBytes);
             ThreadingStats threadingStats = (finalThreadingStats - initialThreadingStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke);
 
             return (gcStats, threadingStats);
diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs
@@ -112,6 +112,7 @@ private static Engine CreateEngine(EngineParameters engineParameters, Job job, A
                 engineParameters.Dummy3Action,
                 idle,
                 main,
+                engineParameters.WorkloadActionSingleInvoke,
                 job,
                 engineParameters.GlobalSetupAction,
                 engineParameters.GlobalCleanupAction,
diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs
@@ -14,6 +14,7 @@ public class EngineParameters
         public IHost Host { get; set; }
         public Action<long> WorkloadActionNoUnroll { get; set; }
         public Action<long> WorkloadActionUnroll { get; set; }
+        public Action WorkloadActionSingleInvoke { get; set; }
         public Action Dummy1Action { get; set; }
         public Action Dummy2Action { get; set; }
         public Action Dummy3Action { get; set; }
diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt
@@ -26,6 +26,7 @@
                 Host = host,
                 WorkloadActionUnroll = instance.WorkloadActionUnroll,
                 WorkloadActionNoUnroll = instance.WorkloadActionNoUnroll,
+                WorkloadActionSingleInvoke = instance.WorkloadActionSingleInvoke,
                 Dummy1Action = instance.Dummy1,
                 Dummy2Action = instance.Dummy2,
                 Dummy3Action = instance.Dummy3,
@@ -149,6 +150,12 @@
                 consumer.Consume(workloadDelegate($PassArguments$)$ConsumeField$);
             }
         }
+
+        private void WorkloadActionSingleInvoke()
+        {
+            $LoadArguments$
+            consumer.Consume(workloadDelegate($PassArguments$)$ConsumeField$);
+        }
         
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
         public $WorkloadMethodReturnType$ $DisassemblerEntryMethodName$()
@@ -208,6 +215,12 @@
             NonGenericKeepAliveWithoutBoxing(result);
         }
 
+        private void WorkloadActionSingleInvoke()
+        {
+            $LoadArguments$
+            NonGenericKeepAliveWithoutBoxing(workloadDelegate($PassArguments$));
+        }
+
         // we must not simply use DeadCodeEliminationHelper.KeepAliveWithoutBoxing<T> because it's generic method
         // and stack-only types like Span<T> can not be generic type arguments http://adamsitnik.com/Span/#span-must-not-be-a-generic-type-argument
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
@@ -273,6 +286,14 @@
             BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias);
         }
 
+        private void WorkloadActionSingleInvoke()
+        {
+            $LoadArguments$
+            ref $WorkloadMethodReturnType$ alias = ref workloadDefaultValueHolder;
+            alias = workloadDelegate($PassArguments$);
+            BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias);
+        }
+
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
         public ref $WorkloadMethodReturnType$ $DisassemblerEntryMethodName$()
         {
@@ -332,6 +353,14 @@
             BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxingReadonly(alias);
         }
 
+        private void WorkloadActionSingleInvoke()
+        {
+            $LoadArguments$
+            ref $WorkloadMethodReturnType$ alias = ref workloadDefaultValueHolder;
+            alias = workloadDelegate($PassArguments$);
+            BenchmarkDotNet.Engines.DeadCodeEliminationHelper.KeepAliveWithoutBoxingReadonly(alias);
+        }
+
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
         public ref readonly $WorkloadMethodReturnType$ $DisassemblerEntryMethodName$()
         {
@@ -381,6 +410,12 @@
             }
         }
 
+        private void WorkloadActionSingleInvoke()
+        {
+            $LoadArguments$
+            workloadDelegate($PassArguments$);
+        }
+
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoOptimization | System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
         public void $DisassemblerEntryMethodName$()
         {
