-
Notifications
You must be signed in to change notification settings - Fork 80
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
#317: Slow performance on discovery / running due to discovering Test…
…Reporters
- Loading branch information
1 parent
01ec27d
commit f9d61b5
Showing
6 changed files
with
104 additions
and
156 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 14 additions & 45 deletions
59
test/test.xunit.runner.visualstudio/RunnerReporterTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,89 +1,58 @@ | ||
| using System.ComponentModel; | ||
| using System; | ||
| using System.Diagnostics; | ||
| using System.Reflection; | ||
| using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; | ||
| using NSubstitute; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
| using Xunit.Runner.Reporters; | ||
| using Xunit.Runner.VisualStudio; | ||
|
|
||
| public class RunnerReporterTests | ||
| { | ||
| public class TestRunnerReporterNotEnabled : IRunnerReporter | ||
| { | ||
| string IRunnerReporter.Description | ||
| => "Not auto-enabled runner"; | ||
|
|
||
| bool IRunnerReporter.IsEnvironmentallyEnabled | ||
| => false; | ||
|
|
||
| string IRunnerReporter.RunnerSwitch | ||
| => "notautoenabled"; | ||
|
|
||
| IMessageSink IRunnerReporter.CreateMessageHandler(IRunnerLogger logger) | ||
| => new NullMessageSink(); | ||
| } | ||
|
|
||
| public class TestRunnerReporter : TestRunnerReporterNotEnabled, IRunnerReporter | ||
| { | ||
| bool IRunnerReporter.IsEnvironmentallyEnabled | ||
| => true; | ||
|
|
||
| string IRunnerReporter.RunnerSwitch | ||
| => null; | ||
| } | ||
|
|
||
| [Fact] | ||
| public void WhenNotUsingAutoReporters_ChoosesDefault() | ||
| { | ||
| using var _ = EnvironmentHelper.NullifyEnvironmentalReporters(); | ||
| var settings = new RunSettings { NoAutoReporters = true }; | ||
|
|
||
| var runnerReporter = VsTestRunner.GetRunnerReporter(null, settings, new[] { Assembly.GetExecutingAssembly().Location }); | ||
| var runnerReporter = VsTestRunner.GetRunnerReporter(null, settings, Array.Empty<string>()); | ||
|
|
||
| Assert.Equal(typeof(DefaultRunnerReporterWithTypes).AssemblyQualifiedName, runnerReporter.GetType().AssemblyQualifiedName); | ||
| } | ||
|
|
||
| [Fact] | ||
| public void WhenUsingAutoReporters_DoesNotChooseDefault() | ||
| { | ||
| using var _ = EnvironmentHelper.NullifyEnvironmentalReporters(); | ||
| Environment.SetEnvironmentVariable("TEAMCITY_PROJECT_NAME", "foo"); // Force TeamCityReporter to surface environmentally | ||
| var settings = new RunSettings { NoAutoReporters = false }; | ||
|
|
||
| var runnerReporter = VsTestRunner.GetRunnerReporter(null, settings, new[] { Assembly.GetExecutingAssembly().Location }); | ||
| var runnerReporter = VsTestRunner.GetRunnerReporter(null, settings, Array.Empty<string>()); | ||
|
|
||
| // We just make sure _an_ auto-reporter was chosen, but we can't rely on which one because this code | ||
| // wil run when we're in CI, and therefore will choose the CI reporter sometimes. It's good enough | ||
| // that we've provide an option above so that the default never gets chosen. | ||
| Assert.NotEqual(typeof(DefaultRunnerReporterWithTypes).AssemblyQualifiedName, runnerReporter.GetType().AssemblyQualifiedName); | ||
| Assert.Equal(typeof(TeamCityReporter).AssemblyQualifiedName, runnerReporter.GetType().AssemblyQualifiedName); | ||
| } | ||
|
|
||
| [Fact] | ||
| public void WhenUsingReporterSwitch_PicksThatReporter() | ||
| { | ||
| var settings = new RunSettings { NoAutoReporters = true, ReporterSwitch = "notautoenabled" }; | ||
| using var _ = EnvironmentHelper.NullifyEnvironmentalReporters(); | ||
| var settings = new RunSettings { NoAutoReporters = true, ReporterSwitch = "json" }; | ||
|
|
||
| var runnerReporter = VsTestRunner.GetRunnerReporter(null, settings, new[] { Assembly.GetExecutingAssembly().Location }); | ||
| var runnerReporter = VsTestRunner.GetRunnerReporter(null, settings, Array.Empty<string>()); | ||
|
|
||
| Assert.Equal(typeof(TestRunnerReporterNotEnabled).AssemblyQualifiedName, runnerReporter.GetType().AssemblyQualifiedName); | ||
| Assert.Equal(typeof(JsonReporter).AssemblyQualifiedName, runnerReporter.GetType().AssemblyQualifiedName); | ||
| } | ||
|
|
||
| [Fact] | ||
| public void WhenRequestedReporterDoesntExist_LogsAndFallsBack() | ||
| { | ||
| using var _ = EnvironmentHelper.NullifyEnvironmentalReporters(); | ||
| var settings = new RunSettings { NoAutoReporters = true, ReporterSwitch = "thisnotavalidreporter" }; | ||
| var logger = Substitute.For<IMessageLogger>(); | ||
| var loggerHelper = new LoggerHelper(logger, new Stopwatch()); | ||
|
|
||
| var runnerReporter = VsTestRunner.GetRunnerReporter(loggerHelper, settings, new[] { Assembly.GetExecutingAssembly().Location }); | ||
| var runnerReporter = VsTestRunner.GetRunnerReporter(loggerHelper, settings, Array.Empty<string>()); | ||
|
|
||
| Assert.Equal(typeof(DefaultRunnerReporterWithTypes).AssemblyQualifiedName, runnerReporter.GetType().AssemblyQualifiedName); | ||
| logger.Received(1).SendMessage(TestMessageLevel.Warning, "[xUnit.net 00:00:00.00] Could not find requested reporter 'thisnotavalidreporter'"); | ||
| } | ||
|
|
||
| [Fact] | ||
| public void VSTestRunnerShouldHaveCategoryAttribute_WithValueManaged() | ||
| { | ||
| var attribute = typeof(VsTestRunner).GetCustomAttribute(typeof(CategoryAttribute)); | ||
| Assert.NotNull(attribute); | ||
| Assert.Equal("managed", (attribute as CategoryAttribute)?.Category); | ||
| } | ||
| } |
51 changes: 51 additions & 0 deletions
51
test/test.xunit.runner.visualstudio/Utility/EnvironmentHelper.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| #nullable enable | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
|
|
||
| static class EnvironmentHelper | ||
| { | ||
| static readonly string[] reporterEnvironmentVariables = | ||
| { | ||
| // AppVeyorReporter | ||
| "APPVEYOR_API_URL", | ||
| // TeamCityReporter | ||
| "TEAMCITY_PROJECT_NAME", | ||
| "TEAMCITY_PROCESS_FLOW_ID", | ||
| // VstsReporter | ||
| "VSTS_ACCESS_TOKEN", | ||
| "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", | ||
| "SYSTEM_TEAMPROJECT", | ||
| "BUILD_BUILDID", | ||
| }; | ||
|
|
||
| public static IDisposable NullifyEnvironmentalReporters() | ||
| { | ||
| var result = new EnvironmentRestorer(reporterEnvironmentVariables); | ||
|
|
||
| foreach (var variable in reporterEnvironmentVariables) | ||
| Environment.SetEnvironmentVariable(variable, null); | ||
|
|
||
| return result; | ||
| } | ||
|
|
||
| public static IDisposable RestoreEnvironment(params string[] variables) => | ||
| new EnvironmentRestorer(variables); | ||
|
|
||
| class EnvironmentRestorer : IDisposable | ||
| { | ||
| Dictionary<string, string?> savedVariables = new(); | ||
|
|
||
| public EnvironmentRestorer(string[] variables) | ||
| { | ||
| foreach (var variable in variables) | ||
| savedVariables[variable] = Environment.GetEnvironmentVariable(variable); | ||
| } | ||
|
|
||
| public void Dispose() | ||
| { | ||
| foreach (var kvp in savedVariables) | ||
| Environment.SetEnvironmentVariable(kvp.Key, kvp.Value); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| using System.ComponentModel; | ||
| using System.Reflection; | ||
| using Xunit; | ||
| using Xunit.Runner.VisualStudio; | ||
|
|
||
| public class VsTestRunnerTests | ||
| { | ||
|
|
||
| [Fact] | ||
| public void VSTestRunnerShouldHaveCategoryAttribute_WithValueManaged() | ||
| { | ||
| var attribute = typeof(VsTestRunner).GetCustomAttribute(typeof(CategoryAttribute)); | ||
|
|
||
| Assert.NotNull(attribute); | ||
| Assert.Equal("managed", (attribute as CategoryAttribute)?.Category); | ||
| } | ||
| } |