From 6a195a3d3e2020f767ff218406c2aead95228678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Jare=C5=A1?= <me@jakubjares.com>
Date: Fri, 20 Jan 2023 09:58:07 +0100
Subject: [PATCH] Remove chutzpah (#4249)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Amaury Levé <amaury.leve@gmail.com>
---
 scripts/build/TestPlatform.Dependencies.props |  1 -
 .../DifferentTestFrameworkSimpleTests.cs      | 11 ++-
 ...rosoft.TestPlatform.AcceptanceTests.csproj |  1 -
 .../DifferentTestFrameworkSimpleTests.cs      | 17 ++---
 .../IntegrationTestBase.cs                    | 13 ++--
 .../NonDll.TestAdapter.csproj                 | 14 ++++
 .../NonDll.TestAdapter/TestAdapter.cs         | 71 +++++++++++++++++++
 test/TestAssets/TestAssets.sln                |  6 ++
 8 files changed, 113 insertions(+), 21 deletions(-)
 create mode 100644 test/TestAssets/NonDll.TestAdapter/NonDll.TestAdapter.csproj
 create mode 100644 test/TestAssets/NonDll.TestAdapter/TestAdapter.cs

diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props
index c6a29db2c7..a675de6daa 100644
--- a/scripts/build/TestPlatform.Dependencies.props
+++ b/scripts/build/TestPlatform.Dependencies.props
@@ -27,7 +27,6 @@
     <NUnit3AdapterVersion>3.11.0</NUnit3AdapterVersion>
     <NUnitConsoleRunnerVersion>3.8.0</NUnitConsoleRunnerVersion>
 
-    <ChutzpahAdapterVersion>4.4.13</ChutzpahAdapterVersion>
     <FluentAssertionsVersion>6.7.0</FluentAssertionsVersion>
 
     <!-- Versions that are used when building projects from TestAssets.sln for compatibility tests. See Invoke-TestAssetsBuild in scripts/build.ps1.
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs
index 2116787346..b23ecd88f2 100644
--- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs
+++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs
@@ -16,16 +16,21 @@ public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase
 {
     [TestMethod]
     [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
-    public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo)
+    public void NonDllRunAllTestExecution(RunnerInfo runnerInfo)
     {
+        // This used to test Chutzpah, to prove that we can run tests that are not shipped in dlls.
+        // But that framework is not fixing vulnerable dependencies for a long time, so we use our custom, test adapter
+        // that simply returns 1 discovered test on discovery, and 1 passed test on execution.
+        // We do not really test that we can run JavaScript tests, but we test that we can trigger tests that are not shipped
+        // in a dll, and pick up the provided test adapter.
         SetTestEnvironment(_testEnvironment, runnerInfo);
         string fileName = "test.js";
         var testJSFileAbsolutePath = Path.Combine(_testEnvironment.TestAssetsPath, fileName);
         string tempPath = Path.Combine(TempDirectory.Path, fileName);
         File.Copy(testJSFileAbsolutePath, tempPath);
-        var arguments = PrepareArguments(tempPath, GetTestAdapterPath(UnitTestFramework.Chutzpah), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path);
+        var arguments = PrepareArguments(tempPath, GetTestAdapterPath(UnitTestFramework.NonDll), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path);
         InvokeVsTest(arguments);
-        ValidateSummaryStatus(1, 1, 0);
+        ValidateSummaryStatus(1, 0, 0);
     }
 
     [TestMethod]
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj
index 9488029108..8bc58eba7c 100644
--- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj
+++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj
@@ -31,7 +31,6 @@
     <ProjectReference Include="$(TestPlatformRoot)test\Microsoft.TestPlatform.TestUtilities\Microsoft.TestPlatform.TestUtilities.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Chutzpah" Version="$(ChutzpahAdapterVersion)" />
     <PackageReference Include="FluentAssertions" Version="$(FluentAssertionsVersion)" />
     <PackageReference Include="Microsoft.TestPlatform.TestAsset.NativeCPP" Version="2.0.0" />
     <PackageReference Include="Microsoft.TestPlatform.QTools.Assets" Version="2.0.0" />
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs
index 0f31084f84..c744db464b 100644
--- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs
+++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs
@@ -116,20 +116,18 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo)
     [TestMethod]
     [TestCategory("Windows-Review")]
     [NetFullTargetFrameworkDataSource]
-    public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo)
+    public void RunTestsWithNonDllAdapter(RunnerInfo runnerInfo)
     {
+        // This used to be test for Chutzpah, but it has long running problem with updating dependencies,
+        // so we test against our own test framework, to ensure that we can run test files that are not using
+        // *.dll as extension.
         SetTestEnvironment(_testEnvironment, runnerInfo);
         Setup();
 
         var jsSource = Path.Combine(_testEnvironment.TestAssetsPath, "test.js");
-
-        // Chuzpah adapter creates _Chutzpah temp files, to give data back from the runner.
-        // But when cleaning up it deletes all the _Chutzpah files, not just the one it owns,
-        // so when we run in parallel, the slower process will never find it's own file, because it was already deleted:
-        // https://github.com/mmanela/chutzpah/issues/812
         var jsInTemp = TempDirectory.CopyFile(jsSource);
 
-        var testAdapterPath = Directory.EnumerateFiles(GetTestAdapterPath(UnitTestFramework.Chutzpah), "*.TestAdapter.dll").ToList();
+        var testAdapterPath = Directory.EnumerateFiles(GetTestAdapterPath(UnitTestFramework.NonDll), "*.TestAdapter.dll").ToList();
         _vstestConsoleWrapper.InitializeExtensions(new List<string>() { testAdapterPath.First() });
 
         _vstestConsoleWrapper.RunTests(
@@ -140,9 +138,8 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo)
         var testCase = _runEventHandler.TestResults.Where(tr => tr.TestCase.DisplayName.Equals("TestMethod1"));
 
         // Assert
-        Assert.AreEqual(2, _runEventHandler.TestResults.Count);
+        Assert.AreEqual(1, _runEventHandler.TestResults.Count);
         Assert.AreEqual(1, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed));
-        Assert.AreEqual(1, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed));
-        Assert.AreEqual(1, testCase.First().TestCase.LineNumber);
+        Assert.AreEqual(0, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed));
     }
 }
diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs
index 15e5674b4a..1acf47134d 100644
--- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs
+++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs
@@ -48,11 +48,10 @@ public class IntegrationTestBase
     private readonly string _testAdapterRelativePath = @"mstest.testadapter\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar);
     private readonly string _nUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build".Replace('\\', Path.DirectorySeparatorChar);
     private readonly string _xUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar);
-    private readonly string _chutzpahTestAdapterRelativePath = @"chutzpah\{0}\tools".Replace('\\', Path.DirectorySeparatorChar);
 
     public enum UnitTestFramework
     {
-        NUnit, XUnit, MSTest, CPP, Chutzpah
+        NUnit, XUnit, MSTest, CPP, NonDll
     }
 
     public IntegrationTestBase()
@@ -574,6 +573,12 @@ protected string GetProjectAssetFullPath(string projectName, string assetName)
 
     protected string GetTestAdapterPath(UnitTestFramework testFramework = UnitTestFramework.MSTest)
     {
+        if (testFramework == UnitTestFramework.NonDll)
+        {
+            var dllPath = _testEnvironment.GetTestAsset("NonDll.TestAdapter.dll", "netstandard2.0");
+            return Path.GetDirectoryName(dllPath)!;
+        }
+
         string adapterRelativePath = string.Empty;
 
         if (testFramework == UnitTestFramework.MSTest)
@@ -588,10 +593,6 @@ protected string GetTestAdapterPath(UnitTestFramework testFramework = UnitTestFr
         {
             adapterRelativePath = string.Format(CultureInfo.InvariantCulture, _xUnitTestAdapterRelativePath, IntegrationTestEnvironment.DependencyVersions["XUnitAdapterVersion"]);
         }
-        else if (testFramework == UnitTestFramework.Chutzpah)
-        {
-            adapterRelativePath = string.Format(CultureInfo.InvariantCulture, _chutzpahTestAdapterRelativePath, IntegrationTestEnvironment.DependencyVersions["ChutzpahAdapterVersion"]);
-        }
 
         return _testEnvironment.GetNugetPackage(adapterRelativePath);
     }
diff --git a/test/TestAssets/NonDll.TestAdapter/NonDll.TestAdapter.csproj b/test/TestAssets/NonDll.TestAdapter/NonDll.TestAdapter.csproj
new file mode 100644
index 0000000000..b0a94b6328
--- /dev/null
+++ b/test/TestAssets/NonDll.TestAdapter/NonDll.TestAdapter.csproj
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <!-- Imports Common TestAssets props. -->
+  <Import Project="..\..\..\scripts\build\TestAssets.props" />
+
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Microsoft.TestPlatform.ObjectModel\Microsoft.TestPlatform.ObjectModel.csproj" />
+  </ItemGroup>
+</Project>
\ No newline at end of file
diff --git a/test/TestAssets/NonDll.TestAdapter/TestAdapter.cs b/test/TestAssets/NonDll.TestAdapter/TestAdapter.cs
new file mode 100644
index 0000000000..b1811d25e4
--- /dev/null
+++ b/test/TestAssets/NonDll.TestAdapter/TestAdapter.cs
@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
+
+namespace NonDll.TestAdapter;
+[FileExtension(".js")]
+[DefaultExecutorUri(Uri)]
+[ExtensionUri(Uri)]
+public class TestAdapter : ITestExecutor, ITestDiscoverer
+{
+    public const string Uri = "executor://nondll.testadapter";
+
+    public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext,
+        IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
+    {
+        var count = 1;
+        foreach (var source in sources)
+        {
+            TestCase testCase = new()
+            {
+                Source = source,
+                CodeFilePath = source,
+                DisplayName = $"Test{count++}",
+                ExecutorUri = new Uri(Uri),
+            };
+            discoverySink.SendTestCase(testCase);
+        }
+    }
+
+    public void Cancel()
+    {
+    }
+
+    public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
+    {
+        foreach (var test in tests)
+        {
+            TestResult testResult = new(test)
+            {
+                Outcome = TestOutcome.Passed,
+            };
+            frameworkHandle.RecordResult(testResult);
+        }
+    }
+
+    public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
+    {
+        var count = 1;
+        foreach (var source in sources)
+        {
+            TestCase testCase = new()
+            {
+                Source = source,
+                CodeFilePath = source,
+                DisplayName = $"Test{count++}",
+                ExecutorUri = new Uri(Uri),
+            };
+            TestResult testResult = new(testCase)
+            {
+                Outcome = TestOutcome.Passed,
+            };
+            frameworkHandle.RecordResult(testResult);
+        }
+    }
+}
diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln
index 37527a5c4a..3db3824c34 100644
--- a/test/TestAssets/TestAssets.sln
+++ b/test/TestAssets/TestAssets.sln
@@ -130,6 +130,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SerializeTestRunTestProject
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MultiHostTestExecutionProject", "MultiHostTestExecutionProject\MultiHostTestExecutionProject.csproj", "{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NonDll.TestAdapter", "NonDll.TestAdapter\NonDll.TestAdapter.csproj", "{429552A4-4C18-4355-94C5-80DC88C48405}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -372,6 +374,10 @@ Global
 		{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}.Release|Any CPU.Build.0 = Release|Any CPU
+		{429552A4-4C18-4355-94C5-80DC88C48405}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{429552A4-4C18-4355-94C5-80DC88C48405}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{429552A4-4C18-4355-94C5-80DC88C48405}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{429552A4-4C18-4355-94C5-80DC88C48405}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE