first version of SchedulingAnalyzer

csoltenborn · Sep 12, 2016 · 4f2f6c0 · 4f2f6c0
1 parent a867792
commit 4f2f6c0
Show file tree

Hide file tree

Showing 8 changed files with 137 additions and 10 deletions.
diff --git a/GoogleTestAdapter/Core.Tests/Runners/SequentialTestRunnerTests.cs b/GoogleTestAdapter/Core.Tests/Runners/SequentialTestRunnerTests.cs
@@ -4,6 +4,7 @@
 using FluentAssertions;
 using GoogleTestAdapter.Helpers;
 using GoogleTestAdapter.Model;
+using GoogleTestAdapter.Scheduling;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 using static GoogleTestAdapter.TestMetadata.TestCategories;
@@ -22,7 +23,7 @@ public void RunTests_CancelingDuringTestExecution_StopsTestExecution()
             List<TestCase> testCasesToRun = TestDataCreator.GetTestCasesOfSampleTests("Crashing.LongRunning", "LongRunningTests.Test2");
 
             var stopwatch = new Stopwatch();
-            var runner = new SequentialTestRunner("", MockFrameworkReporter.Object, TestEnvironment);
+            var runner = new SequentialTestRunner("", MockFrameworkReporter.Object, TestEnvironment, new SchedulingAnalyzer(TestEnvironment));
             var executor = new ProcessExecutor(null, MockLogger.Object);
             var thread = new Thread(() => runner.RunTests(allTestCases, testCasesToRun, "", "", "", false, null, executor));
 

diff --git a/GoogleTestAdapter/Core/Core.csproj b/GoogleTestAdapter/Core/Core.csproj
@@ -57,6 +57,7 @@
     <Compile Include="Helpers\ProcessLauncher.cs" />
     <Compile Include="Helpers\TestProcessLauncher.cs" />
     <Compile Include="Helpers\RegexTraitParser.cs" />
+    <Compile Include="Scheduling\SchedulingAnalyzer.cs" />
     <Compile Include="TestCases\StreamingListTestsParser.cs" />
     <Compile Include="TestCases\ListTestsParser.cs" />
     <Compile Include="TestCases\MethodSignatureCreator.cs" />

diff --git a/GoogleTestAdapter/Core/GoogleTestExecutor.cs b/GoogleTestAdapter/Core/GoogleTestExecutor.cs
@@ -4,6 +4,7 @@
 using GoogleTestAdapter.Model;
 using GoogleTestAdapter.Runners;
 using GoogleTestAdapter.Framework;
+using GoogleTestAdapter.Scheduling;
 
 namespace GoogleTestAdapter
 {
@@ -12,13 +13,15 @@ public class GoogleTestExecutor
     {
 
         private readonly TestEnvironment _testEnvironment;
+        private readonly SchedulingAnalyzer _schedulingAnalyzer;
 
         private ITestRunner _runner;
         private bool _canceled;
 
         public GoogleTestExecutor(TestEnvironment testEnvironment)
         {
             _testEnvironment = testEnvironment;
+            _schedulingAnalyzer = new SchedulingAnalyzer(testEnvironment);
         }
 
 
@@ -39,6 +42,9 @@ public void RunTests(IEnumerable<TestCase> allTestCasesInExecutables, IEnumerabl
             }
 
             _runner.RunTests(allTestCasesInExecutables, testCasesToRunAsArray, solutionDirectory, null, null, isBeingDebugged, launcher, executor);
+
+            if (_testEnvironment.Options.ParallelTestExecution)
+                _schedulingAnalyzer.PrintStatisticsToDebugOutput();
         }
 
         public void Cancel()
@@ -54,11 +60,11 @@ private void ComputeTestRunner(ITestFrameworkReporter reporter, bool isBeingDebu
         {
             if (_testEnvironment.Options.ParallelTestExecution && !isBeingDebugged)
             {
-                _runner = new ParallelTestRunner(reporter, _testEnvironment, solutionDirectory);
+                _runner = new ParallelTestRunner(reporter, _testEnvironment, solutionDirectory, _schedulingAnalyzer);
             }
             else
             {
-                _runner = new PreparingTestRunner(solutionDirectory, reporter, _testEnvironment);
+                _runner = new PreparingTestRunner(solutionDirectory, reporter, _testEnvironment, _schedulingAnalyzer);
                 if (_testEnvironment.Options.ParallelTestExecution && isBeingDebugged)
                 {
                     _testEnvironment.DebugInfo(

diff --git a/GoogleTestAdapter/Core/Model/TestCase.cs b/GoogleTestAdapter/Core/Model/TestCase.cs
@@ -28,6 +28,24 @@ internal string GetTestsuiteName_CommandLineGenerator()
             return FullyQualifiedName.Split('.')[0];
         }
 
+        public override bool Equals(object obj)
+        {
+            var other = obj as TestCase;
+
+            if (other == null)
+                return false;
+
+            return FullyQualifiedName == other.FullyQualifiedName && Source == other.Source;
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = 17;
+            hash = hash * 31 + FullyQualifiedName.GetHashCode();
+            hash = hash * 31 + Source.GetHashCode();
+            return hash;
+        }
+
         public override string ToString()
         {
             return DisplayName;

diff --git a/GoogleTestAdapter/Core/Runners/ParallelTestRunner.cs b/GoogleTestAdapter/Core/Runners/ParallelTestRunner.cs
@@ -14,13 +14,15 @@ public class ParallelTestRunner : ITestRunner
         private readonly TestEnvironment _testEnvironment;
         private readonly List<ITestRunner> _testRunners = new List<ITestRunner>();
         private readonly string _solutionDirectory;
+        private readonly SchedulingAnalyzer _schedulingAnalyzer;
 
 
-        public ParallelTestRunner(ITestFrameworkReporter reporter, TestEnvironment testEnvironment, string solutionDirectory)
+        public ParallelTestRunner(ITestFrameworkReporter reporter, TestEnvironment testEnvironment, string solutionDirectory, SchedulingAnalyzer schedulingAnalyzer)
         {
             _frameworkReporter = reporter;
             _testEnvironment = testEnvironment;
             _solutionDirectory = solutionDirectory;
+            _schedulingAnalyzer = schedulingAnalyzer;
         }
 
 
@@ -68,7 +70,7 @@ private void RunTests(IEnumerable<TestCase> allTestCases, IEnumerable<TestCase>
             int threadId = 0;
             foreach (List<TestCase> testcases in splittedTestCasesToRun)
             {
-                var runner = new PreparingTestRunner(threadId++, _solutionDirectory, _frameworkReporter, _testEnvironment);
+                var runner = new PreparingTestRunner(threadId++, _solutionDirectory, _frameworkReporter, _testEnvironment, _schedulingAnalyzer);
                 _testRunners.Add(runner);
 
                 var thread = new Thread(() => runner.RunTests(allTestCases, testcases, baseDir, null, null, isBeingDebugged, debuggedLauncher, executor));
@@ -82,6 +84,11 @@ private ITestsSplitter GetTestsSplitter(TestCase[] testCasesToRun)
         {
             var serializer = new TestDurationSerializer();
             IDictionary<TestCase, int> durations = serializer.ReadTestDurations(testCasesToRun);
+            foreach (KeyValuePair<TestCase, int> duration in durations)
+            {
+                if (!_schedulingAnalyzer.AddExpectedDuration(duration.Key, duration.Value))
+                    _testEnvironment.DebugWarning("TestCase already in analyzer: " + duration.Key.FullyQualifiedName);
+            }
 
             ITestsSplitter splitter;
             if (durations.Count < testCasesToRun.Length)

diff --git a/GoogleTestAdapter/Core/Runners/PreparingTestRunner.cs b/GoogleTestAdapter/Core/Runners/PreparingTestRunner.cs
@@ -5,6 +5,7 @@
 using GoogleTestAdapter.Helpers;
 using GoogleTestAdapter.Model;
 using GoogleTestAdapter.Framework;
+using GoogleTestAdapter.Scheduling;
 
 namespace GoogleTestAdapter.Runners
 {
@@ -20,19 +21,19 @@ public class PreparingTestRunner : ITestRunner
         private readonly string _solutionDirectory;
 
 
-        public PreparingTestRunner(int threadId, string solutionDirectory, ITestFrameworkReporter reporter, TestEnvironment testEnvironment)
+        public PreparingTestRunner(int threadId, string solutionDirectory, ITestFrameworkReporter reporter, TestEnvironment testEnvironment, SchedulingAnalyzer schedulingAnalyzer)
         {
             _testEnvironment = testEnvironment;
             string threadName = ComputeThreadName(threadId, _testEnvironment.Options.MaxNrOfThreads);
             _threadName = string.IsNullOrEmpty(threadName) ? "" : $"{threadName} ";
             _threadId = Math.Max(0, threadId);
-            _innerTestRunner = new SequentialTestRunner(_threadName, reporter, _testEnvironment);
+            _innerTestRunner = new SequentialTestRunner(_threadName, reporter, _testEnvironment, schedulingAnalyzer);
             _solutionDirectory = solutionDirectory;
         }
 
         public PreparingTestRunner(string solutionDirectory, ITestFrameworkReporter reporter,
-            TestEnvironment testEnvironment)
-            : this(-1, solutionDirectory, reporter, testEnvironment){
+            TestEnvironment testEnvironment, SchedulingAnalyzer schedulingAnalyzer)
+            : this(-1, solutionDirectory, reporter, testEnvironment, schedulingAnalyzer){
         }
 
 

diff --git a/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs b/GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs
@@ -19,13 +19,15 @@ public class SequentialTestRunner : ITestRunner
         private readonly string _threadName;
         private readonly ITestFrameworkReporter _frameworkReporter;
         private readonly TestEnvironment _testEnvironment;
+        private SchedulingAnalyzer _schedulingAnalyzer;
 
 
-        public SequentialTestRunner(string threadName, ITestFrameworkReporter reporter, TestEnvironment testEnvironment)
+        public SequentialTestRunner(string threadName, ITestFrameworkReporter reporter, TestEnvironment testEnvironment, SchedulingAnalyzer schedulingAnalyzer)
         {
             _threadName = threadName;
             _frameworkReporter = reporter;
             _testEnvironment = testEnvironment;
+            _schedulingAnalyzer = schedulingAnalyzer;
         }
 
 
@@ -90,6 +92,12 @@ private void RunTestsFromExecutable(string executable, string workingDir,
                     _testEnvironment.DebugInfo($"{_threadName}Reported {results.Length} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}");
 
                 serializer.UpdateTestDurations(results);
+                foreach (TestResult result in results)
+                {
+
+                    if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
+                        _testEnvironment.DebugWarning("TestCase already in analyzer: " + result.TestCase.FullyQualifiedName);
+                }
             }
         }
 
@@ -124,6 +132,11 @@ private IEnumerable<TestResult> RunTests(string executable, string workingDir, s
                 consoleOutput = new List<string>();
                 new TestDurationSerializer().UpdateTestDurations(splitter.TestResults);
                 _testEnvironment.DebugInfo($"{_threadName}Reported {splitter.TestResults.Count} test results to VS during test execution, executable: '{executable}'");
+                foreach (TestResult result in splitter.TestResults)
+                {
+                    if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int) result.Duration.TotalMilliseconds))
+                        _testEnvironment.LogWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}");
+                }
             }
             else
             {

diff --git a/GoogleTestAdapter/Core/Scheduling/SchedulingAnalyzer.cs b/GoogleTestAdapter/Core/Scheduling/SchedulingAnalyzer.cs
@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using GoogleTestAdapter.Helpers;
+using GoogleTestAdapter.Model;
+
+namespace GoogleTestAdapter.Scheduling
+{
+    public class SchedulingAnalyzer
+    {
+        private class Difference
+        {
+            public TestCase TestCase;
+            public int DifferenceInMs;
+        }
+
+
+        private readonly TestEnvironment _testEnvironment;
+
+        public SchedulingAnalyzer(TestEnvironment testEnvironment)
+        {
+            _testEnvironment = testEnvironment;
+        }
+
+        private ConcurrentDictionary<TestCase, int> ExpectedTestcaseDurations { get; } = new ConcurrentDictionary<TestCase, int>();
+        private ConcurrentDictionary<TestCase, int> ActualTestcaseDurations { get; } = new ConcurrentDictionary<TestCase, int>();
+
+        public bool AddExpectedDuration(TestCase testCase, int duration)
+        {
+            return ExpectedTestcaseDurations.TryAdd(testCase, duration);
+        }
+
+        public bool AddActualDuration(TestCase testCase, int duration)
+        {
+            return ActualTestcaseDurations.TryAdd(testCase, duration);
+        }
+
+        public void PrintStatisticsToDebugOutput()
+        {
+            _testEnvironment.DebugInfo(">>> Scheduling statistics <<<");
+            _testEnvironment.DebugInfo($"# of expected test case durations: {ExpectedTestcaseDurations.Count}");
+            _testEnvironment.DebugInfo($"# of actual test case durations: {ActualTestcaseDurations.Count}");
+            if (ExpectedTestcaseDurations.Count == 0 || ActualTestcaseDurations.Count == 0)
+            {
+                _testEnvironment.DebugInfo("Nothing to report.");
+                return;
+            }
+
+            var differences = new List<Difference>();
+            differences.AddRange(ExpectedTestcaseDurations
+                .Where(ed => ActualTestcaseDurations.ContainsKey(ed.Key))
+                .Select(ed => new Difference
+                    {
+                        TestCase = ed.Key,
+                        DifferenceInMs = ed.Value - ActualTestcaseDurations[ed.Key]
+                    }));
+            differences.Sort((d1, d2) => Math.Abs(d2.DifferenceInMs) - Math.Abs(d1.DifferenceInMs));
+
+            int sumOfAllDifferences = differences.Select(d => d.DifferenceInMs).Sum();
+            double avgDifference = (double) sumOfAllDifferences / differences.Count;
+            double sumOfSquaresOfDifferences = differences.Select(d => (d.DifferenceInMs - avgDifference) * (d.DifferenceInMs - avgDifference)).Sum();
+            double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / differences.Count);
+
+            _testEnvironment.DebugInfo($"{differences.Count} expected durations have been found in actual durations");
+            _testEnvironment.DebugInfo($"Avg difference between expected and actual duration: {avgDifference.ToString("F1", CultureInfo.InvariantCulture)}ms");
+            _testEnvironment.DebugInfo($"Standard deviation: {standardDeviation.ToString("F1", CultureInfo.InvariantCulture)}ms");
+
+            int nrOfWorstDifferences = Math.Min(10, differences.Count);
+            _testEnvironment.DebugInfo($"{nrOfWorstDifferences} worst differences:");
+            for (int i = 0; i < nrOfWorstDifferences; i++)
+            {
+                _testEnvironment.DebugInfo($"Test {differences[i].TestCase.FullyQualifiedName}: Expected {ExpectedTestcaseDurations[differences[i].TestCase]}ms, actual {ActualTestcaseDurations[differences[i].TestCase]}ms");
+            }
+        }
+
+    }
+
+}