Skip to content

Commit

Permalink
Filter out known platform sources (microsoft#4517)
Browse files Browse the repository at this point in the history
  • Loading branch information
nohwnd committed Jun 7, 2023
1 parent 8381658 commit 56c1984
Show file tree
Hide file tree
Showing 3 changed files with 167 additions and 2 deletions.
4 changes: 3 additions & 1 deletion src/vstest.console/CommandLine/CommandLineOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Linq;

using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors;
using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
Expand Down Expand Up @@ -287,7 +288,8 @@ public void AddSource(string source)
string.Format(CultureInfo.CurrentCulture, CommandLineResources.InvalidArgument, source), ex);
}
// Add the matching files to source list
_sources = _sources.Union(matchingFiles).ToList();
var filteredFiles = KnownPlatformSourceFilter.FilterKnownPlatformSources(matchingFiles);
_sources = _sources.Union(filteredFiles).ToList();
}

/// <summary>
Expand Down
67 changes: 66 additions & 1 deletion src/vstest.console/TestPlatformHelpers/TestRequestManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ public void DiscoverTests(
EqtTrace.Info("TestRequestManager.DiscoverTests: Discovery tests started.");

// TODO: Normalize rest of the data on the request as well
discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List<string>();
discoveryPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(discoveryPayload.Sources?.Distinct().ToList());
discoveryPayload.RunSettings ??= "<RunSettings></RunSettings>";

var runsettings = discoveryPayload.RunSettings;
Expand Down Expand Up @@ -274,6 +274,11 @@ public void RunTests(
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
testRunRequestPayload.RunSettings ??= "<RunSettings></RunSettings>";
if (testRunRequestPayload.Sources != null)
{
testRunRequestPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(testRunRequestPayload.Sources);
}

var runsettings = testRunRequestPayload.RunSettings;

if (testRunRequestPayload.TestPlatformOptions != null)
Expand Down Expand Up @@ -1464,3 +1469,63 @@ private static List<string> GetSources(TestRunRequestPayload testRunRequestPaylo
return sources;
}
}

internal static class KnownPlatformSourceFilter
{

// Running tests on AzureDevops, many projects use the default filter
// which includes all *test*.dll, this includes many of the TestPlatform dlls,
// which we cannot run, and don't want to attempt to run.
// The default filter also filters out !*TestAdapter*.dll but it is easy to forget
// so we skip the most used adapters here as well.
private static readonly HashSet<string> KnownPlatformSources = new(new string[]
{
"Microsoft.TestPlatform.CommunicationUtilities.dll",
"Microsoft.TestPlatform.CoreUtilities.dll",
"Microsoft.TestPlatform.CrossPlatEngine.dll",
"Microsoft.TestPlatform.PlatformAbstractions.dll",
"Microsoft.TestPlatform.Utilities.dll",
"Microsoft.VisualStudio.TestPlatform.Common.dll",
"Microsoft.VisualStudio.TestPlatform.ObjectModel.dll",
"testhost.dll",
"Microsoft.TestPlatform.AdapterUtilities.dll",

// NUnit
"NUnit3.TestAdapter.dll",

// XUnit
"xunit.runner.visualstudio.testadapter.dll",
"xunit.runner.visualstudio.dotnetcore.testadapter.dll",

// MSTest
"Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll",
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll",
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll",
"Microsoft.VisualStudio.TestPlatform.TestFramework.dll",
"Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll",
}, StringComparer.OrdinalIgnoreCase);


internal static List<string> FilterKnownPlatformSources(List<string>? sources)
{
if (sources == null)
{
return new List<string>();
}

var filteredSources = new List<string>();
foreach (string source in sources)
{
if (KnownPlatformSources.Contains(Path.GetFileName(source)))
{
EqtTrace.Info($"TestRequestManager.FilterKnownPlatformSources: Known platform dll was provided in sources, removing it '{source}'");
}
else
{
filteredSources.Add(source);
}
}

return filteredSources;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@

using TestPlatform.TestUtilities;
using System.Linq;
using Microsoft.VisualStudio.TestPlatform.Common;
using FluentAssertions;

namespace Microsoft.TestPlatform.AcceptanceTests;

Expand Down Expand Up @@ -420,4 +422,100 @@ public void ExecuteTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(Runne

ExitCodeEquals(1);
}

[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
{
// This is the default filter of AzDo VSTest task:
// testAssemblyVer2: |
// **\*test *.dll
// ! * *\*TestAdapter.dll
// ! * *\obj\**
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
// or deps.json, and fails the run.
SetTestEnvironment(_testEnvironment, runnerInfo);

var testAssemblyPath = _testEnvironment.GetTestAsset("XUTestProject.dll");
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");

string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
var fails = this.StdErrWithWhiteSpace.Split('\n').Where(s => !s.IsNullOrWhiteSpace()).Select(s => s.Trim()).ToList();
fails.Should().HaveCount(2, "because there is 1 failed test, and one message that tests failed.");
fails.Last().Should().Be("Test Run Failed.");
}

[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource()]
public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
{
// This is the default filter of AzDo VSTest task:
// testAssemblyVer2: |
// **\*test *.dll
// ! * *\*TestAdapter.dll
// ! * *\obj\**
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
// or deps.json, and fails the run.
SetTestEnvironment(_testEnvironment, runnerInfo);

var testAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");

string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);

StdErrHasTestRunFailedMessageButNoOtherError();
}

[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource()]
public void RunNunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
{
// This is the default filter of AzDo VSTest task:
// testAssemblyVer2: |
// **\*test *.dll
// ! * *\*TestAdapter.dll
// ! * *\obj\**
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
// or deps.json, and fails the run.
SetTestEnvironment(_testEnvironment, runnerInfo);

var testAssemblyPath = _testEnvironment.GetTestAsset("NUTestProject.dll");
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");

string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);

StdErrHasTestRunFailedMessageButNoOtherError();
}

[TestMethod]
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void RunTestsWhenProvidingJustPlatformDllsFailsTheRun(RunnerInfo runnerInfo)
{
// This is the default filter of AzDo VSTest task:
// testAssemblyVer2: |
// **\*test *.dll
// ! * *\*TestAdapter.dll
// ! * *\obj\**
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
// or deps.json, and fails the run.
SetTestEnvironment(_testEnvironment, runnerInfo);

var xunitAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(xunitAssemblyPath)!, "*test*.dll").Where(f => !f.EndsWith("SimpleTestProject.dll"));

string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Select(s => s.AddDoubleQuote()));
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);

StdErr.Should().Be("No test source files were specified. ", "because all platform files we provided were filtered out");
}
}

0 comments on commit 56c1984

Please sign in to comment.