don't execute long operations more than once per iteration, #736

adamsitnik · adamsitnik · commit cc5834e6e4bf · 2018-05-22T18:04:49.000+02:00
diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs
@@ -41,7 +41,6 @@ public class Engine : IEngine
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineTargetStage targetStage;
         private readonly bool includeMemoryStats;
-        private bool isJitted;
 
         internal Engine(
             IHost host,
@@ -78,22 +77,10 @@ internal Engine(
             targetStage = new EngineTargetStage(this);
         }
 
-        public void Jitting()
-        {
-            // first signal about jitting is raised from auto-generated Program.cs, look at BenchmarkProgram.txt
-            Dummy1Action.Invoke();
-            MainAction.Invoke(1);
-            Dummy2Action.Invoke();
-            IdleAction.Invoke(1);
-            Dummy3Action.Invoke();
-            isJitted = true;
-        }
+        public void Dispose() => GlobalCleanupAction?.Invoke();
 
         public RunResults Run()
         {
-            if (Strategy.NeedsJitting() != isJitted)
-                throw new Exception($"You must{(Strategy.NeedsJitting() ? "" : " not")} call Jitting() first (Strategy = {Strategy})!");
-
             long invokeCount = InvocationCount;
             IReadOnlyList<Measurement> idle = null;
 
diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs
@@ -1,39 +1,85 @@
 using System;
+using BenchmarkDotNet.Horology;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Reports;
 
 namespace BenchmarkDotNet.Engines
 {
-    // TODO: Default instance?
     public class EngineFactory : IEngineFactory
     {
-        public IEngine Create(EngineParameters engineParameters)
+        public IEngine CreateReadyToRun(EngineParameters engineParameters)
         {
-            if (engineParameters.MainAction == null)
-                throw new ArgumentNullException(nameof(engineParameters.MainAction));
+            if (engineParameters.MainSingleAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.MainSingleAction));
+            if (engineParameters.MainMultiAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.MainMultiAction));
             if (engineParameters.Dummy1Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy1Action));
             if (engineParameters.Dummy2Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy2Action));
             if (engineParameters.Dummy3Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy3Action));
-            if (engineParameters.IdleAction == null)
-                throw new ArgumentNullException(nameof(engineParameters.IdleAction));
+            if (engineParameters.IdleSingleAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.IdleSingleAction));
+            if (engineParameters.IdleMultiAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.IdleMultiAction));
             if(engineParameters.TargetJob == null)
                 throw new ArgumentNullException(nameof(engineParameters.TargetJob));
+            
+            engineParameters.GlobalSetupAction?.Invoke();
 
-            return new Engine(
+            var needsJitting = engineParameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, engineParameters.Resolver).NeedsJitting();
+            if (!needsJitting)
+            {
+                // whatever it is, we can not interfere
+                return CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction);
+            }
+
+            var needsPilot = !engineParameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic);
+            if (needsPilot) 
+            {
+                var singleActionEngine = CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction);
+
+                var iterationTime = engineParameters.Resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic);
+                if (ShouldExecuteOncePerIteration(Jit(singleActionEngine), iterationTime))
+                {
+                    var reconfiguredJob = engineParameters.TargetJob.WithInvocationCount(1).WithUnrollFactor(1); // todo: consider if we should set the warmup count to 1!
+
+                    return CreateEngine(engineParameters, reconfiguredJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction);
+                }
+            }
+
+            // it's either a job with explicit configuration or not-very time consuming benchmark, just create the engine, Jit and return
+            var multiActionEngine = CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction);
+                
+            DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine));
+
+            return multiActionEngine;
+        }
+
+        /// <summary>
+        /// returns true if it takes longer than the desired iteration time (0,5s by default) to execute benchmark once
+        /// </summary>
+        private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval iterationTime)
+            => TimeInterval.FromNanoseconds(jit.GetAverageNanoseconds()) > iterationTime;
+
+        private static Measurement Jit(Engine engine)
+            => engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: 1, unrollFactor: 1));
+
+        private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action<long> idle, Action<long> main)
+            => new Engine(
                 engineParameters.Host,
                 engineParameters.Dummy1Action,
                 engineParameters.Dummy2Action,
                 engineParameters.Dummy3Action,
-                engineParameters.IdleAction,
-                engineParameters.MainAction,
-                engineParameters.TargetJob,
+                idle,
+                main,
+                job,
                 engineParameters.GlobalSetupAction,
                 engineParameters.GlobalCleanupAction,
                 engineParameters.IterationSetupAction,
                 engineParameters.IterationCleanupAction,
                 engineParameters.OperationsPerInvoke,
                 engineParameters.MeasureGcStats);
-        }
     }
 }
diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs
@@ -7,11 +7,13 @@ namespace BenchmarkDotNet.Engines
     public class EngineParameters
     {
         public IHost Host { get; set; }
-        public Action<long> MainAction { get; set; }
+        public Action<long> MainSingleAction { get; set; }
+        public Action<long> MainMultiAction { get; set; }
         public Action Dummy1Action { get; set; }
         public Action Dummy2Action { get; set; }
         public Action Dummy3Action { get; set; }
-        public Action<long> IdleAction { get; set; }
+        public Action<long> IdleSingleAction { get; set; }
+        public Action<long> IdleMultiAction { get; set; }
         public Job TargetJob { get; set; } = Job.Default;
         public long OperationsPerInvoke { get; set; } = 1;
         public Action GlobalSetupAction { get; set; } = null;
diff --git a/src/BenchmarkDotNet/Engines/IEngine.cs b/src/BenchmarkDotNet/Engines/IEngine.cs
@@ -6,7 +6,7 @@
 
 namespace BenchmarkDotNet.Engines
 {
-    public interface IEngine
+    public interface IEngine : IDisposable
     {
         [NotNull]
         IHost Host { get; }
@@ -36,12 +36,6 @@ public interface IEngine
 
         Measurement RunIteration(IterationData data);
 
-        /// <summary>
-        /// must perform jitting via warmup calls
-        /// <remarks>is called after first call to GlobalSetup, from the auto-generated benchmark process</remarks>
-        /// </summary>
-        void Jitting();
-
         RunResults Run();
     }
 }
diff --git a/src/BenchmarkDotNet/Engines/IEngineFactory.cs b/src/BenchmarkDotNet/Engines/IEngineFactory.cs
@@ -2,6 +2,6 @@ namespace BenchmarkDotNet.Engines
 {
     public interface IEngineFactory
     {
-        IEngine Create(EngineParameters engineParameters);
+        IEngine CreateReadyToRun(EngineParameters engineParameters);
     }
 }
diff --git a/src/BenchmarkDotNet/Engines/IterationMode.cs b/src/BenchmarkDotNet/Engines/IterationMode.cs
@@ -35,6 +35,11 @@ public enum IterationMode
         /// <summary>
         /// Unknown 
         /// </summary>
-        Unknown
+        Unknown,
+        
+        /// <summary>
+        /// executing benchmark for the purpose of JIT wamup
+        /// </summary>
+        Jit
     }
 }
diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt
@@ -34,23 +34,14 @@
                 MeasureGcStats = $MeasureGcStats$
             };
 
-            var engine = new $EngineFactoryType$().Create(engineParameters);
-
-            instance?.globalSetupAction();
-            instance?.iterationSetupAction();
-
-            if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting())
-				engine.Jitting(); // does first call to main action, must be executed after globalSetup() and iterationSetup()!
-
-            instance?.iterationCleanupAction();
-
-            var results = engine.Run();
-
-            instance?.globalCleanupAction();
+            using (var engine = new $EngineFactoryType$().CreateReadyToRun(engineParameters))
+            {
+                var results = engine.Run();
 
-            host.ReportResults(results); // printing costs memory, do this after runs
+                host.ReportResults(results); // printing costs memory, do this after runs
 
-			instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;)
+                instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;)
+			}
         }
 
         public delegate $IdleMethodReturnTypeName$ IdleDelegate($ArgumentsDefinition$);
diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs
@@ -112,11 +112,11 @@ public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodeg
                 var engineParameters = new EngineParameters
                 {
                     Host = host,
-                    MainAction = mainAction.InvokeMultiple,
+                    MainMultiAction = mainAction.InvokeMultiple,
                     Dummy1Action = dummy1.InvokeSingle,
                     Dummy2Action = dummy2.InvokeSingle,
                     Dummy3Action = dummy3.InvokeSingle,
-                    IdleAction = idleAction.InvokeMultiple,
+                    IdleMultiAction = idleAction.InvokeMultiple,
                     GlobalSetupAction = globalSetupAction.InvokeSingle,
                     GlobalCleanupAction = globalCleanupAction.InvokeSingle,
                     IterationSetupAction = iterationSetupAction.InvokeSingle,
@@ -126,23 +126,14 @@ public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodeg
                     MeasureGcStats = config.HasMemoryDiagnoser()
                 };
 
-                var engine = job
+                using (var engine = job
                     .ResolveValue(InfrastructureMode.EngineFactoryCharacteristic, InfrastructureResolver.Instance)
-                    .Create(engineParameters);
-
-                globalSetupAction.InvokeSingle();
-                iterationSetupAction.InvokeSingle();
-
-                if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting())
-                    engine.Jitting(); // does first call to main action, must be executed after setup()!
-
-                iterationCleanupAction.InvokeSingle();
-
-                var results = engine.Run();
-
-                globalCleanupAction.InvokeSingle();
+                    .CreateReadyToRun(engineParameters))
+                {
+                    var results = engine.Run();
 
-                host.ReportResults(results); // printing costs memory, do this after runs
+                    host.ReportResults(results); // printing costs memory, do this after runs
+                }
             }
         }
     }
diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs
@@ -54,7 +54,7 @@ public void Empty() { }
 
         public class CustomFactory : IEngineFactory
         {
-            public IEngine Create(EngineParameters engineParameters) 
+            public IEngine CreateReadyToRun(EngineParameters engineParameters) 
                 => new CustomEngine
                 {
                     GlobalCleanupAction = engineParameters.GlobalCleanupAction,
@@ -75,6 +75,8 @@ public RunResults Run()
                     default);
             }
 
+            public void Dispose() => GlobalCleanupAction?.Invoke();
+            
             public IHost Host { get; }
             public void WriteLine() { }
             public void WriteLine(string line) { }
@@ -87,7 +89,6 @@ public void WriteLine(string line) { }
             public IResolver Resolver { get; }
 
             public Measurement RunIteration(IterationData data) { throw new NotImplementedException(); }
-            public void Jitting() { }
         }
     }
 }
diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs
diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`
`7`	`7`	`namespace BenchmarkDotNet.Engines`
`8`	`8`	`{`
`9`		`- public interface IEngine`
	`9`	`+ public interface IEngine : IDisposable`
`10`	`10`	`{`
`11`	`11`	`[NotNull]`
`12`	`12`	`IHost Host { get; }`
`@@ -36,12 +36,6 @@ public interface IEngine`
`36`	`36`
`37`	`37`	`Measurement RunIteration(IterationData data);`
`38`	`38`
`39`		`- /// <summary>`
`40`		`- /// must perform jitting via warmup calls`
`41`		`- /// <remarks>is called after first call to GlobalSetup, from the auto-generated benchmark process</remarks>`
`42`		`- /// </summary>`
`43`		`- void Jitting();`
`44`		`-`
`45`	`39`	`RunResults Run();`
`46`	`40`	`}`
`47`	`41`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,6 @@ namespace BenchmarkDotNet.Engines`
`2`	`2`	`{`
`3`	`3`	`public interface IEngineFactory`
`4`	`4`	`{`
`5`		`- IEngine Create(EngineParameters engineParameters);`
	`5`	`+ IEngine CreateReadyToRun(EngineParameters engineParameters);`
`6`	`6`	`}`
`7`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,11 @@ public enum IterationMode`
`35`	`35`	`/// <summary>`
`36`	`36`	`/// Unknown`
`37`	`37`	`/// </summary>`
`38`		`- Unknown`
	`38`	`+ Unknown,`
	`39`	`+`
	`40`	`+ /// <summary>`
	`41`	`+ /// executing benchmark for the purpose of JIT wamup`
	`42`	`+ /// </summary>`
	`43`	`+ Jit`
`39`	`44`	`}`
`40`	`45`	`}`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ public void Empty() { }`
`54`	`54`
`55`	`55`	`public class CustomFactory : IEngineFactory`
`56`	`56`	`{`
`57`		`- public IEngine Create(EngineParameters engineParameters)`
	`57`	`+ public IEngine CreateReadyToRun(EngineParameters engineParameters)`
`58`	`58`	`=> new CustomEngine`
`59`	`59`	`{`
`60`	`60`	`GlobalCleanupAction = engineParameters.GlobalCleanupAction,`
`@@ -75,6 +75,8 @@ public RunResults Run()`
`75`	`75`	`default);`
`76`	`76`	`}`
`77`	`77`
	`78`	`+ public void Dispose() => GlobalCleanupAction?.Invoke();`
	`79`	`+`
`78`	`80`	`public IHost Host { get; }`
`79`	`81`	`public void WriteLine() { }`
`80`	`82`	`public void WriteLine(string line) { }`
`@@ -87,7 +89,6 @@ public void WriteLine(string line) { }`
`87`	`89`	`public IResolver Resolver { get; }`
`88`	`90`
`89`	`91`	`public Measurement RunIteration(IterationData data) { throw new NotImplementedException(); }`
`90`		`- public void Jitting() { }`
`91`	`92`	`}`
`92`	`93`	`}`
`93`	`94`	`}`