Acceptance tests for netcore

TestPlatform.sln

@@ -116,6 +116,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "XUTestProject", "test\TestA
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerfTestProject", "test\TestAssets\PerfTestProject\PerfTestProject.csproj", "{57B182B8-9014-4C6D-B966-B464DE3127D5}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleDataCollector", "test\TestAssets\SimpleDataCollector\SimpleDataCollector.csproj", "{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -630,6 +632,18 @@ Global
 		{57B182B8-9014-4C6D-B966-B464DE3127D5}.Release|x64.Build.0 = Release|x64
 		{57B182B8-9014-4C6D-B966-B464DE3127D5}.Release|x86.ActiveCfg = Release|x86
 		{57B182B8-9014-4C6D-B966-B464DE3127D5}.Release|x86.Build.0 = Release|x86
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Debug|Any CPU.ActiveCfg = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Debug|Any CPU.Build.0 = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Debug|x64.ActiveCfg = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Debug|x64.Build.0 = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Debug|x86.ActiveCfg = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Debug|x86.Build.0 = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Release|x64.ActiveCfg = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Release|x64.Build.0 = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Release|x86.ActiveCfg = Release|Any CPU
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -683,5 +697,6 @@ Global
 		{D68733FF-7520-4906-AE35-B7B9DBAC543F} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
 		{A23E3408-D569-488E-A071-E1B3625C5F09} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
 		{57B182B8-9014-4C6D-B966-B464DE3127D5} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
+		{D62D754C-8F0A-406F-8BA7-E96C6FFA7C7C} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
 	EndGlobalSection
 EndGlobal

dogfood/UnitTestProject/UnitTestProject.csproj

@@ -37,6 +37,9 @@
     <PackageReference Include="Microsoft.NETCore.App">
       <Version>1.0.0</Version>
     </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk">
+      <Version>15.0.0-preview-20161123-03</Version>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'net46' ">

src/Microsoft.TestPlatform.CoreUtilities/Microsoft.TestPlatform.CoreUtilities.csproj

@@ -33,6 +33,9 @@
     </PackageReference>
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'net46' ">
+    <PackageReference Include="Microsoft.Internal.Dia.Interop">
+      <Version>14.0.0</Version>
+    </PackageReference>
     <Reference Include="System.Configuration" />
     <Reference Include="System" />
     <Reference Include="Microsoft.CSharp" />

test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs

@@ -37,6 +37,22 @@ protected static void SetupRunnerDesktopTargetDesktopEnvironment(
             testEnvironment.TargetRuntime = DesktopRunnerTargetRuntime;
         }
 
+        protected static void SetupRunnerCoreTargetCoreEnvironment(
+            IntegrationTestEnvironment testEnvironment)
+        {
+            testEnvironment.RunnerFramework = CoreRunnerFramework;
+            testEnvironment.TargetFramework = CoreTargetFramework;
+            testEnvironment.TargetRuntime = CoreRunnerTargetRuntime;
+        }
+
+        protected static void SetupRunnerDesktopTargetCoreEnvironment(
+            IntegrationTestEnvironment testEnvironment)
+        {
+            testEnvironment.RunnerFramework = DesktopRunnerFramework;
+            testEnvironment.TargetFramework = CoreTargetFramework;
+            testEnvironment.TargetRuntime = DesktopRunnerTargetRuntime;
+        }
+
         protected static string DeriveFrameworkArgValue(IntegrationTestEnvironment testEnvironment)
         {
             string framworkArgValue = string.Empty;
@@ -51,5 +67,30 @@ protected static string DeriveFrameworkArgValue(IntegrationTestEnvironment testE
 
             return framworkArgValue;
         }
+
+        protected bool IsDesktopTargetFramework()
+        {
+            return this.testEnvironment.TargetFramework == AcceptanceTestBase.DesktopTargetFramework;
+        }
+
+        protected bool IsDesktopRunner()
+        {
+            return this.testEnvironment.RunnerFramework == AcceptanceTestBase.DesktopRunnerFramework;
+        }
+
+        protected string GetTargetFramworkForRunsettings()
+        {
+            string targetFramework = string.Empty;
+            if(this.testEnvironment.TargetFramework == DesktopTargetFramework)
+            {
+                targetFramework = "Framework45";
+            }
+            else
+            {
+                targetFramework = "FrameworkCore10";
+            }
+
+            return targetFramework;
+        }
     }
 }

test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs

@@ -10,9 +10,8 @@ namespace Microsoft.TestPlatform.AcceptanceTests
 
     public abstract class AppDomainTests : AcceptanceTestBase
     {
-#if NET46
         [TestMethod]
-        public void RunTestExecutionWithDisableAppDomain()
+        public virtual void RunTestExecutionWithDisableAppDomain()
         {
             var testAppDomainDetailFileName = Path.Combine(Path.GetTempPath(), "appdomain_test.txt");
             var dataCollectorAppDomainDetailFileName = Path.Combine(Path.GetTempPath(), "appdomain_datacollector.txt");
@@ -33,7 +32,6 @@ public void RunTestExecutionWithDisableAppDomain()
             this.ValidateSummaryStatus(1, 1, 1);
             File.Delete(runsettingsFilePath);
         }
-#endif
 
         private static bool IsFilesContentEqual(string filePath1, string filePath2)
         {

test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs

@@ -10,7 +10,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests
     public abstract class DifferentTestFrameworkSimpleTests : AcceptanceTestBase
     {
         [TestMethod]
-        public void ChutzpahRunAllTestExecution()
+        public virtual void ChutzpahRunAllTestExecution()
         {
             var testJSFileAbsolutePath = Path.Combine(this.testEnvironment.TestAssetsPath, "test.js");
             var arguments = PrepareArguments(
@@ -22,8 +22,9 @@ public void ChutzpahRunAllTestExecution()
             this.ValidateSummaryStatus(1, 1, 0);
         }
 
+        [Ignore]
         [TestMethod]
-        public void CPPRunAllTestExecution()
+        public virtual void CPPRunAllTestExecution()
         {
             var assemblyRelativePath =
                 @"microsoft.testplatform.testasset.nativecpp\1.0.0\contentFiles\any\any\Microsoft.TestPlatform.TestAsset.NativeCPP.dll";
@@ -38,7 +39,7 @@ public void CPPRunAllTestExecution()
         }
 
         [TestMethod]
-        public void NUnitRunAllTestExecution()
+        public virtual void NUnitRunAllTestExecution()
         {
             var arguments = PrepareArguments(
                 this.GetAssetFullPath("NUTestProject.dll"),

test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs

@@ -10,7 +10,7 @@ public abstract class DiscoveryTests : AcceptanceTestBase
         [TestMethod]
         public void DiscoverAllTests()
         {
-            this.InvokeVsTestForDiscovery(this.GetSampleTestAssembly(), this.GetTestAdapterPath());
+            this.InvokeVsTestForDiscovery(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue);
             var listOfTests = new string[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" };
             this.ValidateDiscoveredTests(listOfTests);
         }

test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs

@@ -28,7 +28,27 @@ public void RunMultipleTestAssembliesInParallel()
             var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue);
             arguments = string.Concat(arguments, " /Parallel");
             arguments = string.Concat(arguments, " /Platform:x86");
-            var testhostProcessName = "testhost.x86";
+            string testhostProcessName = string.Empty;
+            int expectedNumOfProcessCreated = 0;
+            if (this.IsDesktopTargetFramework())
+            {
+                testhostProcessName = "testhost.x86";
+                expectedNumOfProcessCreated = 2;
+            }
+            else
+            {
+                testhostProcessName = "dotnet";
+                if (this.IsDesktopRunner())
+                {
+                    expectedNumOfProcessCreated = 2;
+                }
+                else
+                {
+                    // Include launcher dotnet.exe
+                    expectedNumOfProcessCreated = 3;
+                }
+            }
+
             var cts = new CancellationTokenSource();
             var numOfProcessCreatedTask = NumberOfProcessLaunchedUtility.NumberOfProcessCreated(
                 cts,
@@ -38,9 +58,9 @@ public void RunMultipleTestAssembliesInParallel()
             cts.Cancel();
 
             Assert.AreEqual(
-                2,
+                expectedNumOfProcessCreated,
                 numOfProcessCreatedTask.Result,
-                $"Number of {testhostProcessName} process created, expected: {2} actual: {numOfProcessCreatedTask.Result}");
+                $"Number of {testhostProcessName} process created, expected: {expectedNumOfProcessCreated} actual: {numOfProcessCreatedTask.Result}");
             this.ValidateSummaryStatus(2, 2, 2);
         }
     }

test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs

@@ -18,8 +18,12 @@ public abstract class PlatformTests : AcceptanceTestBase
         public void RunTestExecutionWithPlatformx64()
         {
             var platformArg = " /Platform:x64";
-            var testhostProcessName = "testhost";
-            this.RunTestExecutionWithPlatform(platformArg, testhostProcessName);
+            string testhostProcessName = string.Empty;
+            int expectedNumOfProcessCreated = 0;
+            string desktopHostProcessName = "testhost";
+
+            SetExpectedParams(ref expectedNumOfProcessCreated, ref testhostProcessName, desktopHostProcessName);
+            this.RunTestExecutionWithPlatform(platformArg, testhostProcessName, expectedNumOfProcessCreated);
         }
 
         /// <summary>
@@ -29,11 +33,36 @@ public void RunTestExecutionWithPlatformx64()
         public void RunTestExecutionWithPlatformx86()
         {
             var platformArg = " /Platform:x86";
-            var testhostProcessName = "testhost.x86";
-            this.RunTestExecutionWithPlatform(platformArg, testhostProcessName);
+            string testhostProcessName = string.Empty;
+            int expectedNumOfProcessCreated = 0;
+            string desktopHostProcessName = "testhost.x86";
+
+            SetExpectedParams(ref expectedNumOfProcessCreated, ref testhostProcessName, desktopHostProcessName);
+            this.RunTestExecutionWithPlatform(platformArg, testhostProcessName, expectedNumOfProcessCreated);
+        }
+
+        private void SetExpectedParams(ref int expectedNumOfProcessCreated, ref string testhostProcessName, string desktopHostProcessName)
+        {
+            if (this.IsDesktopTargetFramework())
+            {
+                testhostProcessName = desktopHostProcessName;
+                expectedNumOfProcessCreated = 1;
+            }
+            else
+            {
+                testhostProcessName = "dotnet";
+                if (this.IsDesktopRunner())
+                {
+                    expectedNumOfProcessCreated = 1;
+                }
+                else
+                {
+                    expectedNumOfProcessCreated = 2;
+                }
+            }
         }
 
-        private void RunTestExecutionWithPlatform(string platformArg, string testhostProcessName)
+        private void RunTestExecutionWithPlatform(string platformArg, string testhostProcessName, int expectedNumOfProcessCreated)
         {
             var arguments = PrepareArguments(
                 this.GetSampleTestAssembly(),
@@ -52,9 +81,9 @@ private void RunTestExecutionWithPlatform(string platformArg, string testhostPro
             cts.Cancel();
 
             Assert.AreEqual(
-                1,
+                expectedNumOfProcessCreated,
                 numOfProcessCreatedTask.Result,
-                $"Number of {testhostProcessName} process created, expected: {1} actual: {numOfProcessCreatedTask.Result}");
+                $"Number of {testhostProcessName} process created, expected: {expectedNumOfProcessCreated} actual: {numOfProcessCreatedTask.Result} args: {arguments} runner path: {this.testEnvironment.GetConsoleRunnerPath()}");
             this.ValidateSummaryStatus(1, 1, 1);
         }
     }

test/Microsoft.TestPlatform.AcceptanceTests/RunnerCoreTargetCore/AppDomainTests.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests.RunnerCoreTargetCore
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class AppDomainTests : AcceptanceTests.AppDomainTests
+    {
+        [TestInitialize]
+        public void SetupEnvironment()
+        {
+            AcceptanceTestBase.SetupRunnerCoreTargetCoreEnvironment(this.testEnvironment);
+        }
+
+        // Disable app domain doesn't support in dotnet core
+        [Ignore]
+        [TestMethod]
+        public override void RunTestExecutionWithDisableAppDomain()
+        {
+
+        }
+    }
+}

test/Microsoft.TestPlatform.AcceptanceTests/RunnerCoreTargetCore/DifferentTestFrameworkSimpleTests.cs

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests.RunnerCoreTargetCore
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class DifferentTestFrameworkSimpleTests : AcceptanceTests.DifferentTestFrameworkSimpleTests
+    {
+        [TestInitialize]
+        public void SetupEnvironment()
+        {
+            AcceptanceTestBase.SetupRunnerCoreTargetCoreEnvironment(this.testEnvironment);
+        }
+
+        // Not supported in dotnet core
+        [Ignore]
+        [TestMethod]
+        public override void ChutzpahRunAllTestExecution()
+        {
+
+        }
+
+        // Not supported in dotnet core
+        [Ignore]
+        [TestMethod]
+        public override void CPPRunAllTestExecution()
+        {
+
+        }
+
+        // Not supported in dotnet core
+        [Ignore]
+        [TestMethod]
+        public override void NUnitRunAllTestExecution()
+        {
+
+        }
+    }
+}

test/Microsoft.TestPlatform.AcceptanceTests/RunnerCoreTargetCore/DiscoveryTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests.RunnerCoreTargetCore
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class DiscoveryTests : AcceptanceTests.DiscoveryTests
+    {
+        [TestInitialize]
+        public void SetupEnvironment()
+        {
+            AcceptanceTestBase.SetupRunnerCoreTargetCoreEnvironment(this.testEnvironment);
+        }
+    }
+}

test/Microsoft.TestPlatform.AcceptanceTests/RunnerCoreTargetCore/ExecutionTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests.RunnerCoreTargetCore
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class ExecutionTests : AcceptanceTests.ExecutionTests
+    {
+        [TestInitialize]
+        public void SetupEnvironment()
+        {
+            AcceptanceTestBase.SetupRunnerCoreTargetCoreEnvironment(this.testEnvironment);
+        }
+    }
+}

test/Microsoft.TestPlatform.AcceptanceTests/RunnerCoreTargetCore/PlatformTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests.RunnerCoreTargetCore
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class PlatformTests : AcceptanceTests.PlatformTests
+   {
+        [TestInitialize]
+        public void SetupEnvironment()
+        {
+            AcceptanceTestBase.SetupRunnerCoreTargetCoreEnvironment(this.testEnvironment);
+        }
+    }
+}