Xunit2 parallelisation

.gitignore

@@ -10,7 +10,7 @@ _ReSharper.*
 *.bak
 install/xunitcontrib-*
 install/UnblockZoneIdentifier.exe
-nuget/*.nupkg
+resharper/nuget/*.nupkg
 nuget/nuget.config
 packages
 

resharper/nuget/xunitcontrib.nuspec

@@ -3,15 +3,23 @@
   <metadata>
     <id>xunitcontrib</id>
     <title>xUnit.net Test Support</title>
-    <version>2.0.0-alpha-20140502</version>
+    <version>2.0.0-alpha-20140718</version>
     <authors>Matt Ellis</authors>
     <owners>Matt Ellis</owners>
     <description>A unit test provider for xUnit.net. Discovers and runs xUnit.net tests. Includes annotations to aid ReSharper inspections and Live Templates to speed up inserting test methods and asserts</description>
     <summary>A unit test provider for xUnit.net</summary>
-    <releaseNotes>&#8226; Initial support for xUnit.net 2</releaseNotes>
+    <releaseNotes>Support for xUnit.net 2.0
+&#8226; Beta 3 support (build 2700)
+&#8226; Shadow copy cache clean up on abort
+
+Known issues:
+&#8226; Parallelisation temporarily disabled to prevent issues with error handling
+&#8226; Runs all tests in a class, not just requested methods, due to xunit1 RunWith compatibility
+&#8226; External annotations not yet implemented for xunit2
+&#8226; Live Templates for Theory use xunit1 namespace</releaseNotes>
     <projectUrl>https://github.com/xunit/resharper-xunit</projectUrl>
     <licenseUrl>https://raw.githubusercontent.com/xunit/resharper-xunit/xunit2/license.txt</licenseUrl>
-    <iconUrl>http://download-codeplex.sec.s-msft.com/Download?ProjectName=xunit&amp;DownloadId=662625</iconUrl>
+    <iconUrl>https://raw.githubusercontent.com/xunit/media/master/logo-512-transparent.png</iconUrl>
     <copyright>Copyright 2014 Matt Ellis</copyright>
     <requireLicenseAcceptance>false</requireLicenseAcceptance>
     <dependencies>

resharper/src/xunitcontrib.runner.resharper.provider/packages.xunitcontrib.runner.resharper.provider.8.2.config

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="JetBrains.ReSharper.SDK" version="8.2.1158" targetFramework="net35" />
-  <package id="xunit.abstractions" version="2.0.0-beta-build2700" targetFramework="net35" />
+  <package id="xunit.abstractions" version="2.0.0-beta-build2716" targetFramework="net35" />
 </packages>

resharper/src/xunitcontrib.runner.resharper.provider/xunitcontrib.runner.resharper.provider.8.2.csproj

@@ -48,7 +48,7 @@
     </Reference>
     <Reference Include="xunit.abstractions, Version=2.0.0.0, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\packages\xunit.abstractions.2.0.0-beta-build2700\lib\net35\xunit.abstractions.dll</HintPath>
+      <HintPath>..\..\packages\xunit.abstractions.2.0.0-beta-build2716\lib\net35\xunit.abstractions.dll</HintPath>
     </Reference>
   </ItemGroup>
   <ItemGroup>

resharper/src/xunitcontrib.runner.resharper.runner/ReSharperRunnerLogger.cs

@@ -11,6 +11,7 @@ namespace XunitContrib.Runner.ReSharper.RemoteRunner
     public class ReSharperRunnerLogger : TestMessageVisitor<ITestAssemblyFinished>
     {
         private const string OneOrMoreChildTestsFailedMessage = "One or more child tests failed";
+
         private readonly RemoteTaskServer server;
         private readonly TaskProvider taskProvider;
 
@@ -20,60 +21,38 @@ public ReSharperRunnerLogger(RemoteTaskServer server, TaskProvider taskProvider)
             this.taskProvider = taskProvider;
         }
 
-        private readonly Stack<TaskInfo> nastySharedState = new Stack<TaskInfo>();
-
-        private void NastySharedState_PushCurrent(TaskInfo task)
-        {
-            nastySharedState.Push(task);
-        }
-
-        private void NastySharedState_PopCurrent(TaskInfo expectedTaskInfo)
-        {
-            var poppedState = nastySharedState.Pop();
-            if (!Equals(poppedState.RemoteTask, expectedTaskInfo.RemoteTask))
-            {
-                server.TaskFinished(expectedTaskInfo.RemoteTask, "Internal (xunit runner): Shared state out of sync!?", TaskResult.Error,
-                    TimeSpan.Zero);
-            }
-        }
-
         protected override bool Visit(ITestClassStarting testClassStarting)
         {
             var classTaskInfo = taskProvider.GetClassTask(testClassStarting.TestClass.Class.Name);
             TaskStarting(classTaskInfo);
             return server.ShouldContinue;
         }
 
-        // Called when a class failure is encountered (i.e., when a fixture from IUseFixture throws an 
-        // exception during construction or System.IDisposable.Dispose)
-        // If the exception happens in the class (fixture) construtor, the child tests are not run, so
-        // we need to mark them all as having failed
-        // xunit2: Seems to be when a class or collection fixture fails
-
-        // Called for catastrophic errors, e.g. ambiguously named methods in xunit1
+        // Called for xunit1:
+        // 1. Catastrophic error, i.e. the test runner throws. The only expected exception
+        //    is for ambiguous method names. Test run is aborted
+        // 2. class failure, i.e. fixture's ctor or dispose throws. Test methods are not run
+        //    if the fixture's ctor throws. Not called for exceptions in class ctor/dispose
+        //    (these are reported as failing test methods, as the class is created for each
+        //     test method)
+        // Not called for xunit2
         protected override bool Visit(IErrorMessage error)
         {
-            // TODO: This is very nasty
-            // OK. I know this will only get called in these scenarios:
-            // 1. Collection fixtures Dispose methods throw exceptions
-            //    Potentially called multiple times, class and methods are still run
-            // 2. Class fixtures Dispose methods throw exceptions
-            //    Potentially called multiple times, methods are still run
-            // 3. xunit1 class fixture ctor or dispose 
-            //    Called once, per class. Methods not run
-            // 4. xunit1 catastrophic error, e.g. ambiguous methods
-            //    Class and methods not run
-            var taskInfo = nastySharedState.Peek();
+            // TODO: This is assuming a lot...
+            var testClasses = from testCase in error.TestCases
+                select testCase.TestMethod.TestClass.Class.Name;
+
+            var taskInfo = taskProvider.GetClassTask(testClasses.First());
 
             string message;
             var exceptions = ConvertExceptions(error, out message);
 
             if (taskInfo.RemoteTask is XunitTestClassTask)
             {
-                var classTaskInfo = (ClassTaskInfo) taskInfo;
+                var classTaskInfo = taskInfo;
                 var methodMessage = string.Format("Class failed in {0}", classTaskInfo.ClassTask.TypeName);
 
-                var methodExceptions = new[] {new TaskException(null, methodMessage, null)};
+                var methodExceptions = new[] { new TaskException(null, methodMessage, null) };
                 foreach (var task in taskProvider.GetDescendants(classTaskInfo))
                 {
                     server.TaskException(task.RemoteTask, methodExceptions);
@@ -86,6 +65,132 @@ protected override bool Visit(IErrorMessage error)
 
             server.TaskException(taskInfo.RemoteTask, exceptions);
 
+            TaskFinished(taskInfo, taskInfo.Message, taskInfo.Result, 0);
+
+            return server.ShouldContinue;
+        }
+
+        protected override bool Visit(ITestAssemblyCleanupFailure testAssemblyCleanupFailure)
+        {
+            Console.WriteLine("ITestAssemblyCleanupFailure");
+
+            // TODO: We can do nothing about this. Report on the root node?
+            return server.ShouldContinue;
+        }
+
+        protected override bool Visit(ITestCollectionCleanupFailure testCollectionCleanupFailure)
+        {
+            // TODO: Add a collection node?
+            var methodMessage = string.Format("Collection cleanup failed in {0}",
+                testCollectionCleanupFailure.TestCollection.DisplayName);
+            HandleCleanupFailure(testCollectionCleanupFailure, testCollectionCleanupFailure.TestCases, methodMessage);
+            return server.ShouldContinue;
+        }
+
+        protected override bool Visit(ITestClassCleanupFailure testClassCleanupFailure)
+        {
+            var methodMessage = string.Format("Class cleanup failed in {0}", testClassCleanupFailure.TestClass.Class.Name);
+            HandleCleanupFailure(testClassCleanupFailure, testClassCleanupFailure.TestCases, methodMessage);
+            return server.ShouldContinue;
+        }
+
+        private void HandleCleanupFailure(IFailureInformation failureInformation, IEnumerable<ITestCase> testCases, string methodMessage)
+        {
+            var testClasses = from testCase in testCases
+                select testCase.TestMethod.TestClass.Class.Name;
+
+            string classMessage;
+            var methodExceptions = new[] { new TaskException(null, methodMessage, null) };
+            var classExceptions = ConvertExceptions(failureInformation, out classMessage);
+
+            foreach (var typeName in testClasses.Distinct())
+            {
+                var taskInfo = taskProvider.GetClassTask(typeName);
+
+                // TODO: Should fail all test cases rather than have to find descendants
+                foreach (var task in taskProvider.GetDescendants(taskInfo))
+                {
+                    server.TaskException(task.RemoteTask, methodExceptions);
+                    server.TaskFinished(task.RemoteTask, methodMessage, TaskResult.Error, 0);
+                }
+
+                server.TaskException(taskInfo.RemoteTask, classExceptions);
+
+                taskInfo.Message = classMessage;
+                taskInfo.Result = TaskResult.Exception;
+
+                if (taskInfo.Finished)
+                {
+                    taskInfo.Finished = false;
+                    TaskFinished(taskInfo, taskInfo.Message, TaskResult.Exception, 0);
+                }
+            }
+
+        }
+
+        protected override bool Visit(ITestMethodCleanupFailure testMethodCleanupFailure)
+        {
+            Console.WriteLine("ITestMethodCleanupFailure");
+
+            // TODO: What causes this? Need to create a test
+
+#if false
+
+            var taskInfo = taskProvider.GetMethodTask(testMethodCleanupFailure.TestClass.Class.Name,
+                testMethodCleanupFailure.TestMethod.Method.Name);
+
+            string message;
+            var exceptions = ConvertExceptions(testMethodCleanupFailure, out message);
+
+            taskInfo.Result = TaskResult.Exception;
+            taskInfo.Message = message;
+
+            server.TaskException(taskInfo.RemoteTask, exceptions);
+
+#endif
+
+            return server.ShouldContinue;
+        }
+
+        protected override bool Visit(ITestCaseCleanupFailure testCaseCleanupFailure)
+        {
+            Console.WriteLine("ITestCaseCleanupFailure");
+
+#if false
+            // TODO: Perhaps look for theory task?
+            var taskInfo = taskProvider.GetMethodTask(testCaseCleanupFailure.TestClass.Class.Name,
+                testCaseCleanupFailure.TestMethod.Method.Name);
+
+            string message;
+            var exceptions = ConvertExceptions(testCaseCleanupFailure, out message);
+
+            taskInfo.Result = TaskResult.Exception;
+            taskInfo.Message = message;
+
+            server.TaskException(taskInfo.RemoteTask, exceptions);
+#endif
+
+            return server.ShouldContinue;
+        }
+
+        protected override bool Visit(ITestCleanupFailure testCleanupFailure)
+        {
+            Console.WriteLine("ITestCleanupFailure");
+
+#if false
+            // TODO: Perhaps look for theory task?
+            var taskInfo = taskProvider.GetMethodTask(testCleanupFailure.TestClass.Class.Name,
+                testCleanupFailure.TestMethod.Method.Name);
+
+            string message;
+            var exceptions = ConvertExceptions(testCleanupFailure, out message);
+
+            taskInfo.Result = TaskResult.Exception;
+            taskInfo.Message = message;
+
+            server.TaskException(taskInfo.RemoteTask, exceptions);
+#endif
+
             return server.ShouldContinue;
         }
 
@@ -106,8 +211,6 @@ protected override bool Visit(ITestMethodStarting testMethodStarting)
             var taskInfo = taskProvider.GetMethodTask(testMethodStarting.TestClass.Class.Name, testMethodStarting.TestMethod.Method.Name);
             TaskStarting(taskInfo);
 
-            // BUG: beta1+2 will report overloaded methods starting/finished twice
-            // Since the method is only keyed on name, it s I'll submit a PR to fix
             taskInfo.Finished = false;
 
             taskInfo.Start();
@@ -223,13 +326,12 @@ private void TaskStarting(TaskInfo taskInfo)
         {
             server.TaskStarting(taskInfo.RemoteTask);
             taskInfo.Started = true;
-
-            NastySharedState_PushCurrent(taskInfo);
         }
 
         private void TaskFinished(TaskInfo taskInfo, string message, TaskResult taskResult, decimal executionTime)
         {
-            NastySharedState_PopCurrent(taskInfo);
+            if (taskInfo.Finished)
+                return;
 
             if (taskInfo.Result != TaskResult.Inconclusive)
                 taskResult = taskInfo.Result;
@@ -249,9 +351,9 @@ private TaskException[] ConvertExceptions(IFailureInformation failure, out strin
             {
                 // Strip out the xunit assert methods from the stack traces by taking
                 // out anything in the Xunit.Assert namespace
-                var stackTraces = failure.StackTraces[i]
+                var stackTraces = failure.StackTraces[i] != null ? failure.StackTraces[i]
                     .Split(new[] {Environment.NewLine}, StringSplitOptions.RemoveEmptyEntries)
-                    .Where(s => !s.Contains("Xunit.Assert")).Join(Environment.NewLine);
+                    .Where(s => !s.Contains("Xunit.Assert")).Join(Environment.NewLine) : string.Empty;
 
                 exceptions.Add(new TaskException(failure.ExceptionTypes[i], failure.Messages[i], stackTraces));
             }

resharper/src/xunitcontrib.runner.resharper.runner/RemoteTaskServer.cs

@@ -56,7 +56,8 @@ public void TaskOutput(RemoteTask remoteTask, string text, TaskOutputType output
 
         public void TaskFinished(RemoteTask remoteTask, string message, TaskResult result, decimal durationInSeconds)
         {
-            TaskFinished(remoteTask, message, result, TimeSpan.FromSeconds((double) durationInSeconds));
+            var timeSpan = durationInSeconds != 0 ? TimeSpan.FromSeconds((double) durationInSeconds) : TimeSpan.Zero;
+            TaskFinished(remoteTask, message, result, timeSpan);
         }
 
         public void TaskFinished(RemoteTask remoteTask, string message, TaskResult result, TimeSpan duration)

resharper/src/xunitcontrib.runner.resharper.runner/TestDiscoveryVisitor.cs

@@ -18,12 +18,5 @@ protected override bool Visit(ITestCaseDiscoveryMessage discovery)
             TestCases.Add(discovery.TestCase);
             return true;
         }
-
-        public override void Dispose()
-        {
-            foreach (var testCase in TestCases)
-                testCase.Dispose();
-            base.Dispose();
-        }
     }
 }

resharper/src/xunitcontrib.runner.resharper.runner/XunitTestRun.cs

@@ -23,10 +23,7 @@ public void RunTests(IEnumerable<ITestCase> testCases)
             var logger = new ReSharperRunnerLogger(server, taskProvider);
             // TODO: Set any execution options?
             // Hmm. testCases is serialised, so can't be an arbitrary IEnumerable<>
-            executor.RunTests(testCases.ToList(), logger, new XunitExecutionOptions
-            {
-                DisableParallelization = true
-            });
+            executor.RunTests(testCases.ToList(), logger, new XunitExecutionOptions());
             logger.Finished.WaitOne();
         }
     }

resharper/src/xunitcontrib.runner.resharper.runner/packages.xunitcontrib.runner.resharper.runner.8.2.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="JetBrains.ReSharper.SDK" version="8.2.1158" targetFramework="net35" />
-  <package id="xunit.abstractions" version="2.0.0-beta-build2700" targetFramework="net35" />
-  <package id="xunit.runner.utility" version="2.0.0-beta-build2700" targetFramework="net35" />
+  <package id="xunit.abstractions" version="2.0.0-beta-build2716" targetFramework="net35" />
+  <package id="xunit.runner.utility" version="2.0.0-beta-build2716" targetFramework="net35" />
 </packages>

resharper/src/xunitcontrib.runner.resharper.runner/xunitcontrib.runner.resharper.runner.8.2.csproj

@@ -40,11 +40,11 @@
     <Reference Include="System.Xml" />
     <Reference Include="xunit.abstractions, Version=2.0.0.0, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\packages\xunit.abstractions.2.0.0-beta-build2700\lib\net35\xunit.abstractions.dll</HintPath>
+      <HintPath>..\..\packages\xunit.abstractions.2.0.0-beta-build2716\lib\net35\xunit.abstractions.dll</HintPath>
     </Reference>
-    <Reference Include="xunit.runner.utility, Version=2.0.0.2700, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+    <Reference Include="xunit.runner.utility, Version=2.0.0.2716, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\packages\xunit.runner.utility.2.0.0-beta-build2700\lib\net35\xunit.runner.utility.dll</HintPath>
+      <HintPath>..\..\packages\xunit.runner.utility.2.0.0-beta-build2716\lib\net35\xunit.runner.utility.dll</HintPath>
     </Reference>
   </ItemGroup>
   <ItemGroup>