introduced test discovery timeout (#90)

csoltenborn · Mar 6, 2017 · 41d7384 · 41d7384
1 parent 905fb53
commit 41d7384
Show file tree

Hide file tree

Showing 13 changed files with 114 additions and 8 deletions.
diff --git a/GoogleTestAdapter/Core.Tests/Core.Tests.csproj b/GoogleTestAdapter/Core.Tests/Core.Tests.csproj
@@ -92,6 +92,7 @@
     <Compile Include="Helpers\UtilsTests.cs" />
     <Compile Include="Runners\CommandLineGeneratorTests.cs" />
     <Compile Include="Runners\SequentialTestRunnerTests.cs" />
+    <Compile Include="TestCases\TestCaseFactoryTests.cs" />
     <Compile Include="TestCases\ListTestsParserTests.cs" />
     <Compile Include="TestResults\StreamingStandardOutputTestResultParserTests.cs" />
     <Compile Include="TestResults\StandardOutputTestResultParserTests.cs" />

diff --git a/GoogleTestAdapter/Core.Tests/TestCases/TestCaseFactoryTests.cs b/GoogleTestAdapter/Core.Tests/TestCases/TestCaseFactoryTests.cs
@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using FluentAssertions;
+using GoogleTestAdapter.Model;
+using GoogleTestAdapter.Tests.Common;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using static GoogleTestAdapter.Tests.Common.TestMetadata.TestCategories;
+
+namespace GoogleTestAdapter.TestCases
+{
+
+    [TestClass]
+    public class TestCaseFactoryTests : TestsBase
+    {
+
+        [TestMethod]
+        [TestCategory(Unit)]
+        public void CreateTestCases_DiscoveryTimeoutIsExceeded_DiscoveryIsCanceledAndCancelationIsLogged()
+        {
+            MockOptions.Setup(o => o.TestDiscoveryTimeoutInSeconds).Returns(1);
+            MockOptions.Setup(o => o.ParseSymbolInformation).Returns(false);
+
+            var reportedTestCases = new List<TestCase>();
+            var stopWatch = Stopwatch.StartNew();
+            var factory = new TestCaseFactory(TestResources.TenSecondsWaiter, MockLogger.Object, TestEnvironment.Options, null);
+            var returnedTestCases = factory.CreateTestCases(testCase => reportedTestCases.Add(testCase));
+            stopWatch.Stop();
+
+            reportedTestCases.Should().BeEmpty();
+            returnedTestCases.Should().BeEmpty();
+            stopWatch.Elapsed.Should().BeGreaterOrEqualTo(TimeSpan.FromSeconds(1));
+            stopWatch.Elapsed.Should().BeLessThan(TimeSpan.FromSeconds(2));
+            MockLogger.Verify(o => o.LogError(It.Is<string>(s => s.Contains(TestResources.TenSecondsWaiter))), Times.Once);
+            MockLogger.Verify(o => o.DebugError(It.Is<string>(s => s.Contains(Path.GetFileName(TestResources.TenSecondsWaiter)))), Times.Once);
+        }
+
+    }
+
+}
diff --git a/GoogleTestAdapter/Core/Settings/IGoogleTestAdapterSettings.cs b/GoogleTestAdapter/Core/Settings/IGoogleTestAdapterSettings.cs
@@ -27,6 +27,7 @@ public interface IGoogleTestAdapterSettings
         bool? ShuffleTests { get; set; }
         int? ShuffleTestsSeed { get; set; }
         string TestDiscoveryRegex { get; set; }
+        int? TestDiscoveryTimeoutInSeconds { get; set; }
         string WorkingDir { get; set; }
         string PathExtension { get; set; }
         string BatchForTestSetup { get; set; }
@@ -58,6 +59,7 @@ public static void GetUnsetValuesFrom(this IGoogleTestAdapterSettings self, IGoo
             self.ShuffleTests = self.ShuffleTests ?? other.ShuffleTests;
             self.ShuffleTestsSeed = self.ShuffleTestsSeed ?? other.ShuffleTestsSeed;
             self.TestDiscoveryRegex = self.TestDiscoveryRegex ?? other.TestDiscoveryRegex;
+            self.TestDiscoveryTimeoutInSeconds = self.TestDiscoveryTimeoutInSeconds ?? other.TestDiscoveryTimeoutInSeconds;
             self.WorkingDir = self.WorkingDir ?? other.WorkingDir;
             self.PathExtension = self.PathExtension ?? other.PathExtension;
             self.BatchForTestSetup = self.BatchForTestSetup ?? other.BatchForTestSetup;

diff --git a/GoogleTestAdapter/Core/Settings/RunSettings.cs b/GoogleTestAdapter/Core/Settings/RunSettings.cs
@@ -24,6 +24,9 @@ public RunSettings(string projectRegex)
         public virtual string TestDiscoveryRegex { get; set; }
         public bool ShouldSerializeTestDiscoveryRegex() { return TestDiscoveryRegex != null; }
 
+        public virtual int? TestDiscoveryTimeoutInSeconds { get; set; }
+        public bool ShouldSerializeTestDiscoveryTimeoutInSeconds() { return TestDiscoveryTimeoutInSeconds != null; }
+
         public virtual string WorkingDir { get; set; }
         public bool ShouldSerializeWorkingDir() { return WorkingDir != null; }
 

diff --git a/GoogleTestAdapter/Core/Settings/SettingsWrapper.cs b/GoogleTestAdapter/Core/Settings/SettingsWrapper.cs
@@ -259,6 +259,23 @@ public static string ReplacePlaceholders(string userParameters, string executabl
         public virtual string TestDiscoveryRegex => _currentSettings.TestDiscoveryRegex ?? OptionTestDiscoveryRegexDefaultValue;
 
 
+        public const string OptionTestDiscoveryTimeoutInSeconds = "Test discovery timeout in s";
+        public const int OptionTestDiscoveryTimeoutInSecondsDefaultValue = 30;
+        public const string OptionTestDiscoveryTimeoutInSecondsDescription =
+            "Number of seconds after which test discovery will be assumed to have failed. 0: Infinite timeout";
+
+        public virtual int TestDiscoveryTimeoutInSeconds {
+            get
+            {
+                int timeout = _currentSettings.TestDiscoveryTimeoutInSeconds ?? OptionTestDiscoveryTimeoutInSecondsDefaultValue;
+                if (timeout < 0)
+                    timeout = OptionTestDiscoveryTimeoutInSecondsDefaultValue;
+
+                return timeout == 0 ? int.MaxValue : timeout;
+            }
+        }
+
+
         public const string OptionWorkingDir = "Working directory";
         public const string OptionWorkingDirDefaultValue = ExecutableDirPlaceholder;
         public const string OptionWorkingDirDescription =

diff --git a/GoogleTestAdapter/Core/TestCases/TestCaseFactory.cs b/GoogleTestAdapter/Core/TestCases/TestCaseFactory.cs
@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using GoogleTestAdapter.Common;
 using GoogleTestAdapter.DiaResolver;
 using GoogleTestAdapter.Helpers;
@@ -13,7 +14,7 @@
 namespace GoogleTestAdapter.TestCases
 {
 
-    internal class TestCaseFactory
+    public class TestCaseFactory
     {
         private readonly ILogger _logger;
         private readonly SettingsWrapper _settings;
@@ -103,13 +104,34 @@ private IList<TestCase> NewCreateTestcases(Action<TestCase> reportTestCase, List
 
             try
             {
-                var executor = new ProcessExecutor(null, _logger);
-                int processExitCode = executor.ExecuteCommandBlocking(
-                    _executable,
-                    GoogleTestConstants.ListTestsOption.Trim(),
-                    "",
-                    _settings.GetPathExtension(_executable),
-                    lineAction);
+                int processExitCode = ProcessExecutor.ExecutionFailed;
+                ProcessExecutor executor = null;
+                var listAndParseTestsTask = new Task(() =>
+                {
+                    executor = new ProcessExecutor(null, _logger);
+                    processExitCode = executor.ExecuteCommandBlocking(
+                        _executable,
+                        GoogleTestConstants.ListTestsOption.Trim(),
+                        "",
+                        _settings.GetPathExtension(_executable),
+                        lineAction);
+                }, TaskCreationOptions.AttachedToParent);
+                listAndParseTestsTask.Start();
+
+                if (!listAndParseTestsTask.Wait(TimeSpan.FromSeconds(_settings.TestDiscoveryTimeoutInSeconds)))
+                {
+                    executor?.Cancel();
+
+                    string dir = Path.GetDirectoryName(_executable);
+                    string file = Path.GetFileName(_executable);
+                    string cdToWorkingDir = $@"cd ""{dir}""";
+                    string listTestsCommand = $"{file} {GoogleTestConstants.ListTestsOption.Trim()}";
+
+                    _logger.LogError($"Test discovery was cancelled after {_settings.TestDiscoveryTimeoutInSeconds}s for executable {_executable}");
+                    _logger.DebugError($"Test whether the following commands can be executed sucessfully on the command line (make sure all required binaries are on the PATH):{Environment.NewLine}{cdToWorkingDir}{Environment.NewLine}{listTestsCommand}");
+
+                    return new List<TestCase>();
+                }
 
                 if (!CheckProcessExitCode(processExitCode, standardOutput))
                     return new List<TestCase>();

diff --git a/GoogleTestAdapter/Resources/AllTestSettings.gta.runsettings b/GoogleTestAdapter/Resources/AllTestSettings.gta.runsettings
@@ -19,6 +19,7 @@
         <ShuffleTests>false</ShuffleTests>
         <ShuffleTestsSeed>0</ShuffleTestsSeed>
         <TestDiscoveryRegex>.*Tests.*.exe</TestDiscoveryRegex>
+        <TestDiscoveryTimeoutInSeconds>30</TestDiscoveryTimeoutInSeconds>
         <WorkingDir>${SolutionDir}</WorkingDir>
         <PathExtension>C:\fooDir;C:\barDir</PathExtension>
         <TraitsRegexesAfter />

diff --git a/GoogleTestAdapter/Tests.Common/Resources/TestData/misc/TenSecondsWaiter.exe b/GoogleTestAdapter/Tests.Common/Resources/TestData/misc/TenSecondsWaiter.exe
diff --git a/GoogleTestAdapter/Tests.Common/TestResources.cs b/GoogleTestAdapter/Tests.Common/TestResources.cs
@@ -21,6 +21,8 @@ public static class TestResources
         public const string Results0Batch = @"Tests\Returns0.bat";
         public const string Results1Batch = @"Tests\Returns1.bat";
 
+        public const string TenSecondsWaiter = TestdataDir + @"misc\TenSecondsWaiter.exe";
+
         public const int NrOfSampleTests = 88;
         public const string SampleTests = SampleTestsSolutionDir + @"Debug\Tests_gta.exe";
         public const string SampleTestsRelease = SampleTestsSolutionDir + @"Release\Tests_gta.exe";

diff --git a/GoogleTestAdapter/Tests.Common/Tests.Common.csproj b/GoogleTestAdapter/Tests.Common/Tests.Common.csproj
@@ -123,6 +123,9 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <Content Include="Resources\TestData\misc\TenSecondsWaiter.exe">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="Resources\TestData\SampleResult1 _Broken_InvalidStatusAttribute.xml">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

diff --git a/GoogleTestAdapter/Tests.Common/TestsBase.cs b/GoogleTestAdapter/Tests.Common/TestsBase.cs
@@ -43,6 +43,8 @@ public static void SetupOptions(Mock<SettingsWrapper> mockOptions)
             mockOptions.Setup(o => o.CheckCorrectUsage(It.IsAny<string>())).Callback(() => { });
             mockOptions.Setup(o => o.Clone()).Returns(mockOptions.Object);
 
+            mockOptions.Setup(o => o.TestDiscoveryTimeoutInSeconds)
+                .Returns(SettingsWrapper.OptionTestDiscoveryTimeoutInSecondsDefaultValue);
             mockOptions.Setup(o => o.TraitsRegexesBefore).Returns(new List<RegexTraitPair>());
             mockOptions.Setup(o => o.TraitsRegexesAfter).Returns(new List<RegexTraitPair>());
             mockOptions.Setup(o => o.TestNameSeparator).Returns(SettingsWrapper.OptionTestNameSeparatorDefaultValue);

diff --git a/GoogleTestAdapter/VsPackage/GoogleTestExtensionOptionsPage.cs b/GoogleTestAdapter/VsPackage/GoogleTestExtensionOptionsPage.cs
@@ -159,6 +159,7 @@ private RunSettings GetRunSettingsFromOptionPages()
             {
                 PrintTestOutput = _generalOptions.PrintTestOutput,
                 TestDiscoveryRegex = _generalOptions.TestDiscoveryRegex,
+                TestDiscoveryTimeoutInSeconds = _generalOptions.TestDiscoveryTimeoutInSeconds,
                 WorkingDir = _generalOptions.WorkingDir,
                 PathExtension = _generalOptions.PathExtension,
                 TraitsRegexesBefore = _generalOptions.TraitsRegexesBefore,

diff --git a/GoogleTestAdapter/VsPackage/OptionsPages/GeneralOptionsDialogPage.cs b/GoogleTestAdapter/VsPackage/OptionsPages/GeneralOptionsDialogPage.cs
@@ -20,6 +20,16 @@ public string TestDiscoveryRegex
         }
         private string _testDiscoveryRegex = SettingsWrapper.OptionTestDiscoveryRegexDefaultValue;
 
+        [Category(SettingsWrapper.CategoryTestExecutionName)]
+        [DisplayName(SettingsWrapper.OptionTestDiscoveryTimeoutInSeconds)]
+        [Description(SettingsWrapper.OptionTestDiscoveryTimeoutInSecondsDescription)]
+        public int TestDiscoveryTimeoutInSeconds
+        {
+            get { return _testDiscoveryTimeoutInSeconds; }
+            set { SetAndNotify(ref _testDiscoveryTimeoutInSeconds, value); }
+        }
+        private int _testDiscoveryTimeoutInSeconds = SettingsWrapper.OptionTestDiscoveryTimeoutInSecondsDefaultValue;
+
         [Category(SettingsWrapper.CategoryTestExecutionName)]
         [DisplayName(SettingsWrapper.OptionWorkingDir)]
         [Description(SettingsWrapper.OptionWorkingDirDescription)]