diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AbortionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AbortionTests.cs index 490e0be93d..54d24b27e0 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AbortionTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AbortionTests.cs @@ -9,15 +9,15 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class AbortionTests : BaseAcceptanceTests +public class AbortionTests : AcceptanceTestBase { private const string AssetName = "Abort"; - private readonly AbortionTestsFixture _abortionTestsFixture; + private readonly TestAssetFixture _testAssetFixture; - public AbortionTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, AbortionTestsFixture abortionTestsFixture) - : base(testExecutionContext, acceptanceFixture) + public AbortionTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _abortionTestsFixture = abortionTestsFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] @@ -30,10 +30,10 @@ public async Task AbortWithCTRLPlusC_TestHost_Succeeded(string tfm) return; } - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_abortionTestsFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync(); - testHostResult.AssertHasExitCode(ExitCodes.TestSessionAborted); + testHostResult.AssertExitCodeIs(ExitCodes.TestSessionAborted); // We check only in netcore for netfx is now showing in CI every time, the same behavior in local something works sometime nope. // Manual test works pretty always as expected, looks like the implementation is different, we care more on .NET Core. @@ -46,40 +46,27 @@ public async Task AbortWithCTRLPlusC_TestHost_Succeeded(string tfm) } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class AbortionTestsFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _testAsset; + public string TargetAssetPath => GetAssetPath(AssetName); - public AbortionTestsFixture(AcceptanceFixture acceptanceFixture) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _acceptanceFixture = acceptanceFixture; - } - - public string TargetAssetPath => _testAsset is null ? throw new ArgumentNullException(nameof(TestAsset)) : _testAsset.TargetAssetPath; - - public async Task InitializeAsync(InitializationContext context) - { - _testAsset = await TestAsset.GenerateAssetAsync(AssetName, Sources.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - // We expect the same semantic for Linux, the test setup is not cross and we're using specific // Windows API because this gesture is not easy xplat. if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - return; + yield break; } - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); + yield return (AssetName, AssetName, Sources.PatchTargetFrameworks(TargetFrameworks.All)); } - public void Dispose() => _testAsset?.Dispose(); - } - - private const string Sources = """ + private const string Sources = """ #file Abort.csproj - tfms + $TargetFrameworks$ Exe true enable @@ -179,6 +166,6 @@ internal class Capabilities : ITestFrameworkCapabilities { ITestFrameworkCapability[] ICapabilities.Capabilities => Array.Empty(); } - """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DiagnosticTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DiagnosticTests.cs index 7dcd04506f..3847528e69 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DiagnosticTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DiagnosticTests.cs @@ -10,26 +10,25 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class DiagnosticTests : BaseAcceptanceTests +public class DiagnosticTests : AcceptanceTestBase { private const string AssetName = "DiagnosticTest"; - private readonly BuildFixture _buildFixture; + private readonly TestAssetFixture _testAssetFixture; - public DiagnosticTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, - BuildFixture buildFixture) - : base(testExecutionContext, acceptanceFixture) + public DiagnosticTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _buildFixture = buildFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_WhenDiagnosticIsSpecified_ReportIsGeneratedInDefaultLocation(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic"); await AssertDiagnosticReportWasGeneratedAsync(testHostResult, diagPathPattern); @@ -38,10 +37,10 @@ public async Task Diag_WhenDiagnosticIsSpecified_ReportIsGeneratedInDefaultLocat [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_WhenDiagnosticAndOutputFilePrefixAreSpecified_ReportIsGeneratedInDefaultLocation(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"abcd_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic --diagnostic-output-fileprefix abcd"); await AssertDiagnosticReportWasGeneratedAsync(testHostResult, diagPathPattern); @@ -50,12 +49,12 @@ public async Task Diag_WhenDiagnosticAndOutputFilePrefixAreSpecified_ReportIsGen [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_WhenDiagnosticAndOutputDirectoryAreSpecified_ReportIsGeneratedInSpecifiedLocation(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, "test1"); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, "test1"); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); Assert.IsTrue(Directory.CreateDirectory(diagPath).Exists); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--diagnostic --diagnostic-output-directory {diagPath}"); await AssertDiagnosticReportWasGeneratedAsync(testHostResult, diagPathPattern); @@ -64,12 +63,12 @@ public async Task Diag_WhenDiagnosticAndOutputDirectoryAreSpecified_ReportIsGene [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_WhenDiagnosticAndOutputFilePrefixAndOutputDirectoryAreSpecified_ReportIsGeneratedInSpecifiedLocation(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, "test2"); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, "test2"); string diagPathPattern = Path.Combine(diagPath, @"abcde_.*.diag").Replace(@"\", @"\\"); Assert.IsTrue(Directory.CreateDirectory(diagPath).Exists); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--diagnostic --diagnostic-output-fileprefix abcde --diagnostic-output-directory {diagPath}"); await AssertDiagnosticReportWasGeneratedAsync(testHostResult, diagPathPattern); @@ -78,40 +77,40 @@ public async Task Diag_WhenDiagnosticAndOutputFilePrefixAndOutputDirectoryAreSpe [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_WhenDiagnosticOutputFilePrefixButNotDiagnosticIsSpecified_ReportGenerationFails(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic-output-fileprefix cccc"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); testHostResult.AssertOutputContains("'--diagnostic-output-fileprefix' requires '--diagnostic' to be provided"); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_WhenDiagnosticOutputDirectoryButNotDiagnosticIsSpecified_ReportGenerationFails(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic-output-directory cccc"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); testHostResult.AssertOutputContains("'--diagnostic-output-directory' requires '--diagnostic' to be provided"); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_WhenDiagnosticFilePrefixAndDiagnosticOutputDirectoryButNotDiagnosticAreSpecified_ReportGenerationFails(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic-output-fileprefix aaaa --diagnostic-output-directory cccc"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); testHostResult.AssertOutputContains("'--diagnostic-output-directory' requires '--diagnostic' to be provided"); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_EnableWithEnvironmentVariables_Succeeded(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -125,10 +124,10 @@ public async Task Diag_EnableWithEnvironmentVariables_Succeeded(string tfm) [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_EnableWithEnvironmentVariables_Verbosity_Succeeded(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -143,10 +142,10 @@ public async Task Diag_EnableWithEnvironmentVariables_Verbosity_Succeeded(string [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_EnableWithEnvironmentVariables_CustomPrefix_Succeeded(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"MyPrefix_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -161,10 +160,10 @@ public async Task Diag_EnableWithEnvironmentVariables_CustomPrefix_Succeeded(str [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_EnableWithEnvironmentVariables_SynchronousWrite_Succeeded(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -179,7 +178,7 @@ public async Task Diag_EnableWithEnvironmentVariables_SynchronousWrite_Succeeded [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Diag_EnableWithEnvironmentVariables_Disable_Succeeded(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( "--diagnostic", @@ -187,17 +186,17 @@ public async Task Diag_EnableWithEnvironmentVariables_Disable_Succeeded(string t { { EnvironmentVariableConstants.TESTINGPLATFORM_DIAGNOSTIC, "0" }, }); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); testHostResult.AssertOutputDoesNotContain("Diagnostic file"); testHostResult = await testHost.ExecuteAsync("--diagnostic"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); testHostResult.AssertOutputContains("Diagnostic file"); } private async Task AssertDiagnosticReportWasGeneratedAsync(TestHostResult testHostResult, string diagPathPattern, string level = "Information", string flushType = "async") { - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); string outputPattern = $""" Diagnostic file \(level '{level}' with {flushType} flush\): {diagPathPattern} @@ -226,34 +225,20 @@ private async Task AssertDiagnosticReportWasGeneratedAsync(TestHostResul } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class BuildFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _testAsset; + public string TargetAssetPath => GetAssetPath(AssetName); - public string TargetAssetPath => _testAsset!.TargetAssetPath; - - public BuildFixture(AcceptanceFixture acceptanceFixture) - { - _acceptanceFixture = acceptanceFixture; - } - - public async Task InitializeAsync(InitializationContext context) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _testAsset = await TestAsset.GenerateAssetAsync( - AssetName, - TestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); + yield return (AssetName, AssetName, TestCode.PatchTargetFrameworks(TargetFrameworks.All)); } - public void Dispose() => _testAsset?.Dispose(); - } - - private const string TestCode = """ + private const string TestCode = """ #file DiagnosticTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -290,4 +275,5 @@ public void TestMethod1() global using Microsoft.Testing.Framework; global using Microsoft.Testing.Platform.Extensions; """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/EnvironmentVariablesConfigurationProviderTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/EnvironmentVariablesConfigurationProviderTests.cs index 4130b8ee7e..0208724a9d 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/EnvironmentVariablesConfigurationProviderTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/EnvironmentVariablesConfigurationProviderTests.cs @@ -7,15 +7,14 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public sealed class EnvironmentVariablesConfigurationProviderTests : BaseAcceptanceTests +public sealed class EnvironmentVariablesConfigurationProviderTests : AcceptanceTestBase { - private readonly EnvironmentVariablesConfigurationProviderFixture _environmentVariablesConfigurationProviderFixture; + private readonly TestAssetFixture _testAssetFixture; - public EnvironmentVariablesConfigurationProviderTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, - EnvironmentVariablesConfigurationProviderFixture environmentVariablesConfigurationProviderFixture) - : base(testExecutionContext, acceptanceFixture) + public EnvironmentVariablesConfigurationProviderTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _environmentVariablesConfigurationProviderFixture = environmentVariablesConfigurationProviderFixture; + _testAssetFixture = testAssetFixture; } private const string AssetName = "EnvironmentVariablesConfigurationProvider"; @@ -23,7 +22,7 @@ public EnvironmentVariablesConfigurationProviderTests(ITestExecutionContext test [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task DefaultEnabledEnvironmentVariablesConfiguration_SetEnvironmentVariable_ShouldSucceed(string currentTfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_environmentVariablesConfigurationProviderFixture.TargetAssetPath, AssetName, currentTfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, currentTfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -32,13 +31,13 @@ public async Task DefaultEnabledEnvironmentVariablesConfiguration_SetEnvironment { "MyValue", "MyVal" }, { "MYENVVAR__MYPROP1__MYPROP2", "MyVal" }, }); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task EnabledEnvironmentVariablesConfiguration_SetEnvironmentVariable_ShouldSucceed(string currentTfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_environmentVariablesConfigurationProviderFixture.TargetAssetPath, AssetName, currentTfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, currentTfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -47,50 +46,39 @@ public async Task EnabledEnvironmentVariablesConfiguration_SetEnvironmentVariabl { "MyValue", "MyVal" }, { "MYENVVAR__MYPROP1__MYPROP2", "MyVal" }, }); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task DisabledEnvironmentVariablesConfiguration_SetEnvironmentVariable_ShouldSucceed(string currentTfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_environmentVariablesConfigurationProviderFixture.TargetAssetPath, AssetName, currentTfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, currentTfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() { { "EnableConfigurationSource", "false" }, }); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class EnvironmentVariablesConfigurationProviderFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) + : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _testAsset; + public string TargetAssetPath => GetAssetPath(AssetName); - public string TargetAssetPath => _testAsset is null ? throw new ArgumentNullException(nameof(TestAsset)) : _testAsset.TargetAssetPath; - - public EnvironmentVariablesConfigurationProviderFixture(AcceptanceFixture acceptanceFixture) - { - _acceptanceFixture = acceptanceFixture; - } - - public async Task InitializeAsync(InitializationContext context) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _testAsset = await TestAsset.GenerateAssetAsync(AssetName, Sources.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); + yield return (AssetName, AssetName, Sources.PatchTargetFrameworks(TargetFrameworks.All)); } - public void Dispose() => _testAsset?.Dispose(); - } - - private const string Sources = """ + private const string Sources = """ #file EnvironmentVariablesConfigurationProvider.csproj - tfms + $TargetFrameworks$ Exe true enable @@ -219,4 +207,5 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context) } } """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs index f968db7a08..fe9317f8d8 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs @@ -7,28 +7,24 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class ExecutionTests : BaseAcceptanceTests +public class ExecutionTests : AcceptanceTestBase { private const string AssetName = "ExecutionTests"; - private const BuildConfiguration AssetBuildConfiguration = BuildConfiguration.Release; + private readonly TestAssetFixture _testAssetFixture; - private readonly BuildFixture _buildFixture; - - public ExecutionTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, BuildFixture buildFixture) - : base(testExecutionContext, acceptanceFixture) + public ExecutionTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _buildFixture = buildFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Exec_WhenListTestsIsSpecified_AllTestsAreFound(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--list-tests", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--list-tests"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string OutputPattern = """ The following Tests are available: @@ -43,12 +39,10 @@ public async Task Exec_WhenListTestsIsSpecified_AllTestsAreFound(string tfm) [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Exec_WhenOnlyAssetNameIsSpecified_AllTestsAreRun(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - null, - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync(); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string OutputPattern = $""" Passed! - Failed: 0, Passed: 4, Skipped: 0, Total: 4, Duration: .+s - {AssetName}.+$ @@ -59,12 +53,10 @@ public async Task Exec_WhenOnlyAssetNameIsSpecified_AllTestsAreRun(string tfm) [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Exec_WhenListTestsAndFilterAreSpecified_OnlyFilteredTestsAreFound(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--list-tests --treenode-filter \"/ExecutionTests/ExecutionTests/UnitTest1/TestMethod*\"", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--list-tests --treenode-filter \"/ExecutionTests/ExecutionTests/UnitTest1/TestMethod*\""); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string OutputPattern = """ The following Tests are available: @@ -78,12 +70,10 @@ public async Task Exec_WhenListTestsAndFilterAreSpecified_OnlyFilteredTestsAreFo [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Exec_WhenFilterIsSpecified_OnlyFilteredTestsAreRun(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - $"--treenode-filter \"/ExecutionTests/ExecutionTests/UnitTest1/TestMethod*\"", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--treenode-filter \"/ExecutionTests/ExecutionTests/UnitTest1/TestMethod*\""); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string OutputPattern = $""" Passed! - Failed: 0, Passed: 3, Skipped: 0, Total: 3, Duration: .+s - {AssetName}.+$ @@ -94,12 +84,10 @@ public async Task Exec_WhenFilterIsSpecified_OnlyFilteredTestsAreRun(string tfm) [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Exec_WhenMinimumExpectedTestsIsSpecifiedAndEnoughTestsRun_ResultIsOk(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--minimum-expected-tests 4", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--minimum-expected-tests 4"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string OutputPattern = $""" Passed! - Failed: 0, Passed: 4, Skipped: 0, Total: 4, Duration: .+s - {AssetName}.+$ @@ -110,12 +98,10 @@ public async Task Exec_WhenMinimumExpectedTestsIsSpecifiedAndEnoughTestsRun_Resu [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Exec_WhenMinimumExpectedTestsIsSpecifiedAndNotEnoughTestsRun_ResultIsNotOk(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--minimum-expected-tests 5", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--minimum-expected-tests 5"); - testHostResult.AssertHasExitCode(ExitCodes.MinimumExpectedTestsPolicyViolation); + testHostResult.AssertExitCodeIs(ExitCodes.MinimumExpectedTestsPolicyViolation); const string OutputPattern = $""" Minimum expected tests policy violation, tests ran 4, minimum expected 5 - Failed: 0, Passed: 4, Skipped: 0, Total: 4, Duration: .+s - {AssetName}.+$ @@ -126,46 +112,30 @@ public async Task Exec_WhenMinimumExpectedTestsIsSpecifiedAndNotEnoughTestsRun_R [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Exec_WhenListTestsAndMinimumExpectedTestsAreSpecified_DiscoveryFails(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--list-tests --minimum-expected-tests 4", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--list-tests --minimum-expected-tests 4"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); const string OutputPattern = "Error: '--list-tests' and '--minimum-expected-tests' are incompatible options"; Assert.That(testHostResult.StandardOutput.Contains(OutputPattern), $"Output of the test host is:\n{testHostResult}"); } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class BuildFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _testAsset; - - public string TargetAssetPath => _testAsset!.TargetAssetPath; - - public BuildFixture(AcceptanceFixture acceptanceFixture) - { - _acceptanceFixture = acceptanceFixture; - } + public string TargetAssetPath => GetAssetPath(AssetName); - public async Task InitializeAsync(InitializationContext context) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _testAsset = await TestAsset.GenerateAssetAsync( - AssetName, - TestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c {AssetBuildConfiguration}", _acceptanceFixture.NuGetGlobalPackagesFolder); + yield return (AssetName, AssetName, TestCode.PatchTargetFrameworks(TargetFrameworks.All)); } - public void Dispose() => _testAsset?.Dispose(); - } - - private const string TestCode = """ + private const string TestCode = """ #file ExecutionTests.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -217,4 +187,5 @@ public void FilteredOutTest() global using Microsoft.Testing.Framework; global using Microsoft.Testing.Platform.Extensions; """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpTests.cs index 382b999b39..ac6be1ecac 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpTests.cs @@ -7,32 +7,30 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class HelpTests : BaseAcceptanceTests +public class HelpTests : AcceptanceTestBase { - private readonly HelpAssetsFixture _helpAssetsFixture; + private readonly TestAssetFixture _testAssetFixture; - public HelpTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, HelpAssetsFixture helpAssetsFixture) - : base(testExecutionContext, acceptanceFixture) + public HelpTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _helpAssetsFixture = helpAssetsFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Help_WhenNoExtensionRegistered_OutputDefaultHelpContent(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_helpAssetsFixture.NoExtensionTargetAssetPath, HelpAssetsFixture.NoExtensionAssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--help", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.NoExtensionTargetAssetPath, TestAssetFixture.NoExtensionAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--help"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string RegexMatchPattern = $""" Microsoft\(R\) Testing Platform Execution Command Line Tool Version: \d+\.\d+\.\d+(-.*)? RuntimeInformation: .+ Copyright\(c\) Microsoft Corporation\.  All rights reserved\. -Usage {HelpAssetsFixture.NoExtensionAssetName}.* \[option providers\] \[extension option providers\] +Usage {TestAssetFixture.NoExtensionAssetName}.* \[option providers\] \[extension option providers\] Execute a .NET Test Application\. Options: --diagnostic Enable the diagnostic logging\. The default log level is 'Information'\. The file will be written in the output directory with the name log_\[MMddHHssfff\]\.diag @@ -57,14 +55,14 @@ public async Task Help_WhenNoExtensionRegisteredAndUnknownOptionIsSpecified_Outp { const string UnknownOption = "aaa"; - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_helpAssetsFixture.NoExtensionTargetAssetPath, HelpAssetsFixture.NoExtensionAssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.NoExtensionTargetAssetPath, TestAssetFixture.NoExtensionAssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"-{UnknownOption}"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); const string RegexMatchPattern = $""" Unknown option '--{UnknownOption}' -Usage {HelpAssetsFixture.NoExtensionAssetName}.* \[option providers\] \[extension option providers\] +Usage {TestAssetFixture.NoExtensionAssetName}.* \[option providers\] \[extension option providers\] Execute a \.NET Test Application\. Options: --diagnostic Enable the diagnostic logging\. The default log level is 'Information'\. The file will be written in the output directory with the name log_\[MMddHHssfff\]\.diag @@ -87,19 +85,17 @@ Execute a \.NET Test Application\. [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Help_WhenMSTestExtensionRegistered_OutputHelpContentOfRegisteredExtension(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_helpAssetsFixture.MSTestTargetAssetPath, HelpAssetsFixture.MSTestAssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--help", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.MSTestTargetAssetPath, TestAssetFixture.MSTestAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--help"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string RegexMatchPattern = $""" Microsoft\(R\) Testing Platform Execution Command Line Tool Version: \d+\.\d+\.\d+(-.*)? RuntimeInformation: .+ Copyright\(c\) Microsoft Corporation\.  All rights reserved\. -Usage {HelpAssetsFixture.MSTestAssetName}.* \[option providers\] \[extension option providers\] +Usage {TestAssetFixture.MSTestAssetName}.* \[option providers\] \[extension option providers\] Execute a .NET Test Application\. Options: --diagnostic Enable the diagnostic logging\. The default log level is 'Information'\. The file will be written in the output directory with the name log_\[MMddHHssfff\]\.diag @@ -121,59 +117,26 @@ Execute a .NET Test Application\. } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class HelpAssetsFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { public const string NoExtensionAssetName = "NoExtensionHelpTest"; public const string MSTestAssetName = "MSTestHelpTest"; - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _noExtensionTestAsset; - private TestAsset? _mstestTestAsset; + public string NoExtensionTargetAssetPath => GetAssetPath(NoExtensionAssetName); - public string NoExtensionTargetAssetPath => _noExtensionTestAsset!.TargetAssetPath; + public string MSTestTargetAssetPath => GetAssetPath(MSTestAssetName); - public string MSTestTargetAssetPath => _mstestTestAsset!.TargetAssetPath; - - public HelpAssetsFixture(AcceptanceFixture acceptanceFixture) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _acceptanceFixture = acceptanceFixture; + yield return (NoExtensionAssetName, NoExtensionAssetName, NoExtensionHelpTestCode.PatchTargetFrameworks(TargetFrameworks.All)); + yield return (MSTestAssetName, MSTestAssetName, MSTestCode.PatchTargetFrameworks(TargetFrameworks.All)); } - public async Task InitializeAsync(InitializationContext context) - { - await Task.WhenAll( - GenerateNoExtensionTestAsset(), - GenerateMSTestTestAsset()); - - async Task GenerateNoExtensionTestAsset() - { - _noExtensionTestAsset = await TestAsset.GenerateAssetAsync( - NoExtensionAssetName, - NoExtensionHelpTestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_noExtensionTestAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } - - async Task GenerateMSTestTestAsset() - { - _mstestTestAsset = await TestAsset.GenerateAssetAsync( - MSTestAssetName, - MSTestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_mstestTestAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } - } - - public void Dispose() - { - _noExtensionTestAsset?.Dispose(); - _mstestTestAsset?.Dispose(); - } - } - - private const string NoExtensionHelpTestCode = """ + private const string NoExtensionHelpTestCode = """ #file NoExtensionHelpTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -211,11 +174,11 @@ public void TestMethod1() global using Microsoft.Testing.Platform.Extensions; """; - private const string MSTestCode = """ + private const string MSTestCode = """ #file MSTestHelpTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -256,4 +219,5 @@ public void TestMethod1() global using Microsoft.Testing.Platform.Extensions; global using Microsoft.VisualStudio.TestTools.UnitTesting; """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceAssert.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceAssert.cs index 71ec8c3844..6b40e2aea1 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceAssert.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceAssert.cs @@ -7,9 +7,12 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; internal static class AcceptanceAssert { - public static void AssertHasExitCode(this TestHostResult testHostResult, int exitCode) + public static void AssertExitCodeIs(this TestHostResult testHostResult, int exitCode) => Assert.That(exitCode == testHostResult.ExitCode, GenerateFailedAssertionMessage(testHostResult)); + public static void AssertExitCodeIsNot(this TestHostResult testHostResult, int exitCode) + => Assert.That(exitCode != testHostResult.ExitCode, GenerateFailedAssertionMessage(testHostResult)); + public static void AssertOutputMatchesRegex(this TestHostResult testHostResult, string pattern) => Assert.That(Regex.IsMatch(testHostResult.StandardOutput, pattern), GenerateFailedAssertionMessage(testHostResult)); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceFixture.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceFixture.cs index 8422fd5987..b80a7529e0 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceFixture.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceFixture.cs @@ -6,11 +6,8 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestFixture(TestFixtureSharingStrategy.PerTestApplication)] public sealed class AcceptanceFixture : IDisposable { - private readonly TempDirectory _nuGetGlobalPackagesFolder = new("NugetCache"); - - public string NuGetGlobalPackagesFolder - => _nuGetGlobalPackagesFolder.DirectoryPath; + public TempDirectory NuGetGlobalPackagesFolder { get; } = new("NugetCache"); public void Dispose() - => _nuGetGlobalPackagesFolder.Dispose(); + => NuGetGlobalPackagesFolder.Dispose(); } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/BaseAcceptance.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs similarity index 79% rename from test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/BaseAcceptance.cs rename to test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs index a16a8350b9..89e71a1613 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/BaseAcceptance.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs @@ -11,19 +11,19 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; /// At the moment are static because we need to share them between perclass/id fixtures and /// it's not supported at the moment. /// -public abstract partial class BaseAcceptanceTests : TestBase +public abstract partial class AcceptanceTestBase : TestBase { [GeneratedRegex("^(.*?)\\.(?=(?:[0-9]+\\.){2,}[0-9]+(?:-[a-z]+)?\\.nupkg)(.*?)\\.nupkg$")] - private static partial Regex ParseNugetPacakgeFileName(); + private static partial Regex ParseNuGetPackageFileNameRegex(); internal static string RID { get; private set; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "win-x64" : "linux-x64"; public static string MSTestCurrentVersion { get; private set; } - static BaseAcceptanceTests() + static AcceptanceTestBase() { var mstestTestFrameworkPackage = Path.GetFileName(Directory.GetFiles(Constants.ArtifactsPackagesShipping, "MSTest.TestFramework*.nupkg", SearchOption.AllDirectories).Single()); - Match match = ParseNugetPacakgeFileName().Match(mstestTestFrameworkPackage); + Match match = ParseNuGetPackageFileNameRegex().Match(mstestTestFrameworkPackage); if (!match.Success) { throw new InvalidOperationException("Package version not found"); @@ -32,14 +32,11 @@ static BaseAcceptanceTests() MSTestCurrentVersion = match.Groups[2].Value; } - protected BaseAcceptanceTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture) + protected AcceptanceTestBase(ITestExecutionContext testExecutionContext) : base(testExecutionContext) { - AcceptanceFixture = acceptanceFixture; } - public AcceptanceFixture AcceptanceFixture { get; } - internal static IEnumerable> GetBuildMatrixTfmBuildConfiguration() { foreach (TestArgumentsEntry tfm in TargetFrameworks.All) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/InfoTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/InfoTests.cs index 048d58f2e2..50fb1b5192 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/InfoTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/InfoTests.cs @@ -7,23 +7,23 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class InfoTests : BaseAcceptanceTests +public class InfoTests : AcceptanceTestBase { - private readonly InfoAssetsFixture _infoAssetsFixture; + private readonly TestAssetFixture _testAssetFixture; - public InfoTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, InfoAssetsFixture infoAssetsFixture) - : base(testExecutionContext, acceptanceFixture) + public InfoTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _infoAssetsFixture = infoAssetsFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Info_WhenNoExtensionRegistered_OutputDefaultInfoContent(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_infoAssetsFixture.NoExtensionTargetAssetPath, InfoAssetsFixture.NoExtensionAssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.NoExtensionTargetAssetPath, TestAssetFixture.NoExtensionAssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--info"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); string regexMatchPattern = $""" Microsoft\(R\) Testing Platform Execution Command Line Tool @@ -34,7 +34,7 @@ public async Task Info_WhenNoExtensionRegistered_OutputDefaultInfoContent(string Version: \d+\.\d+\.\d+(-.*)? Dynamic Code Supported: True Runtime information: .+({Environment.NewLine} Runtime location: .+)? - Test module: .+{InfoAssetsFixture.NoExtensionAssetName}.* + Test module: .+{TestAssetFixture.NoExtensionAssetName}.* Built-in command line providers: PlatformCommandLineProvider Name: Platform command line provider @@ -133,12 +133,10 @@ There are no registered tools\. [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Info_WhenMSTestExtensionRegistered_OutputInfoContentOfRegisteredExtension(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_infoAssetsFixture.MSTestTargetAssetPath, InfoAssetsFixture.MSTestAssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync( - "--info", - new Dictionary { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.MSTestTargetAssetPath, TestAssetFixture.MSTestAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--info"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); const string RegexMatchPattern = $""" Registered command line providers: @@ -161,55 +159,26 @@ public async Task Info_WhenMSTestExtensionRegistered_OutputInfoContentOfRegister } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class InfoAssetsFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { public const string NoExtensionAssetName = "NoExtensionInfoTest"; public const string MSTestAssetName = "MSTestInfoTest"; - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _noExtensionTestAsset; - private TestAsset? _mstestTestAsset; - - public InfoAssetsFixture(AcceptanceFixture acceptanceFixture) - { - _acceptanceFixture = acceptanceFixture; - } - - public string NoExtensionTargetAssetPath => _noExtensionTestAsset!.TargetAssetPath; + public string NoExtensionTargetAssetPath => GetAssetPath(NoExtensionAssetName); - public string MSTestTargetAssetPath => _mstestTestAsset!.TargetAssetPath; + public string MSTestTargetAssetPath => GetAssetPath(MSTestAssetName); - public async Task InitializeAsync(InitializationContext context) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - await Task.WhenAll( - GenerateNoExtensionTestAsset(), - GenerateMSTestTestAsset()); - - async Task GenerateNoExtensionTestAsset() - { - _noExtensionTestAsset = await TestAsset.GenerateAssetAsync( - NoExtensionAssetName, - NoExtensionTestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_noExtensionTestAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } - - async Task GenerateMSTestTestAsset() - { - _mstestTestAsset = await TestAsset.GenerateAssetAsync( - MSTestAssetName, - MSTestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_mstestTestAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } + yield return (NoExtensionAssetName, NoExtensionAssetName, NoExtensionTestCode.PatchTargetFrameworks(TargetFrameworks.All)); + yield return (MSTestAssetName, MSTestAssetName, MSTestCode.PatchTargetFrameworks(TargetFrameworks.All)); } - public void Dispose() => _noExtensionTestAsset?.Dispose(); - } - - private const string NoExtensionTestCode = """ + private const string NoExtensionTestCode = """ #file NoExtensionInfoTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -247,11 +216,11 @@ public void TestMethod1() global using Microsoft.Testing.Platform.Extensions; """; - private const string MSTestCode = """ + private const string MSTestCode = """ #file MSTestInfoTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -292,4 +261,5 @@ public void TestMethod1() global using Microsoft.Testing.Platform.Extensions; global using Microsoft.VisualStudio.TestTools.UnitTesting; """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSTestRunnerTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSTestRunnerTests.cs index 6424c27afe..99e159258b 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSTestRunnerTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSTestRunnerTests.cs @@ -6,13 +6,13 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class MSTestRunnerTests : BaseAcceptanceTests +public class MSTestRunnerTests : AcceptanceTestBase { private readonly AcceptanceFixture _acceptanceFixture; private const string AssetName = "MSTestProject"; public MSTestRunnerTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture) - : base(testExecutionContext, acceptanceFixture) + : base(testExecutionContext) { _acceptanceFixture = acceptanceFixture; } @@ -23,18 +23,18 @@ public async Task EnableMSTestRunner_True_Will_Run_Standalone(string tfm, BuildC using TestAsset generator = await TestAsset.GenerateAssetAsync( AssetName, CurrentTemplateSourceCode - .PatchCodeWithRegularExpression("tfm", tfm) - .PatchCodeWithRegularExpression("mstestversion", MSTestCurrentVersion) - .PatchCodeWithRegularExpression("enablemstestrunner", "true") - .PatchCodeWithRegularExpression("outputtype", "Exe"), + .PatchCodeWithReplace("$TargetFramework$", tfm) + .PatchCodeWithReplace("$MSTestVersion$", MSTestCurrentVersion) + .PatchCodeWithReplace("$EnableMSTestRunner$", "true") + .PatchCodeWithReplace("$OutputType$", "Exe"), addPublicFeeds: true); string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog"); - var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder); + var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); compilationResult = await DotnetCli.RunAsync( $"build -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}", - _acceptanceFixture.NuGetGlobalPackagesFolder, failIfReturnValueIsNotZero: false); + _acceptanceFixture.NuGetGlobalPackagesFolder.Path, failIfReturnValueIsNotZero: false); var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); - var testHostResult = await testHost.ExecuteAsync(environmentVariables: new Dictionary() { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + var testHostResult = await testHost.ExecuteAsync(); testHostResult.AssertOutputContains("Passed! - Failed: 0, Passed: 1, Skipped: 0, Total: 1"); } @@ -44,19 +44,19 @@ public async Task EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_Fro using TestAsset generator = await TestAsset.GenerateAssetAsync( AssetName, CurrentTemplateSourceCode - .PatchCodeWithRegularExpression("tfm", tfm) - .PatchCodeWithRegularExpression("mstestversion", MSTestCurrentVersion) - .PatchCodeWithRegularExpression("enablemstestrunner", "false") - .PatchCodeWithRegularExpression("outputtype", "Exe"), + .PatchCodeWithReplace("$TargetFramework$", tfm) + .PatchCodeWithReplace("$MSTestVersion$", MSTestCurrentVersion) + .PatchCodeWithReplace("$EnableMSTestRunner$", "false") + .PatchCodeWithReplace("$OutputType$", "Exe"), addPublicFeeds: true); string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog"); - var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder); + var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); try { - compilationResult = await DotnetCli.RunAsync($"build -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder); + compilationResult = await DotnetCli.RunAsync($"build -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path); var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); - var testHostResult = await testHost.ExecuteAsync(environmentVariables: new Dictionary() { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } }); + var testHostResult = await testHost.ExecuteAsync(); Assert.AreEqual(string.Empty, testHostResult.StandardOutput); } catch (Exception ex) @@ -76,17 +76,17 @@ public async Task EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_Fro - tfm + $TargetFramework$ false true - outputtype - enablemstestrunner + $OutputType$ + $EnableMSTestRunner$ - - + + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs index ce9193bc9c..2a85bd7895 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs @@ -7,10 +7,10 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class NoBannerTests : BaseAcceptanceTests +public class NoBannerTests : AcceptanceTestBase { private const string AssetName = "NoBannerTest"; - private readonly BuildFixture _buildFixture; + private readonly TestAssetFixture _testAssetFixture; private readonly string _bannerRegexMatchPattern = """ \s*Microsoft\(R\) Testing Platform Execution Command Line Tool.* \s*Version:.* @@ -18,26 +18,26 @@ public class NoBannerTests : BaseAcceptanceTests \s*Copyright\(c\) Microsoft Corporation[.]  All rights reserved[.].* """; - public NoBannerTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, BuildFixture buildFixture) - : base(testExecutionContext, acceptanceFixture) + public NoBannerTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _buildFixture = buildFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task UsingNoBanner_TheBannerDoesNotAppear(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--no-banner"); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); testHostResult.AssertOutputDoesNotMatchRegex(_bannerRegexMatchPattern); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task UsingNoBanner_InTheEnvironmentVars_TheBannerDoesNotAppear(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -45,14 +45,14 @@ public async Task UsingNoBanner_InTheEnvironmentVars_TheBannerDoesNotAppear(stri { "TESTINGPLATFORM_NOBANNER", "true" }, }); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); testHostResult.AssertOutputDoesNotMatchRegex(_bannerRegexMatchPattern); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task UsingDotnetNoLogo_InTheEnvironmentVars_TheBannerDoesNotAppear(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( null, new Dictionary() @@ -60,49 +60,35 @@ public async Task UsingDotnetNoLogo_InTheEnvironmentVars_TheBannerDoesNotAppear( { "DOTNET_NOLOGO", "true" }, }); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); testHostResult.AssertOutputDoesNotMatchRegex(_bannerRegexMatchPattern); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task WithoutUsingNoBanner_TheBannerAppears(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync(); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); testHostResult.AssertOutputMatchesRegex(_bannerRegexMatchPattern); } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class BuildFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _testAsset; + public string TargetAssetPath => GetAssetPath(AssetName); - public string TargetAssetPath => _testAsset!.TargetAssetPath; - - public BuildFixture(AcceptanceFixture acceptanceFixture) - { - _acceptanceFixture = acceptanceFixture; - } - - public async Task InitializeAsync(InitializationContext context) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _testAsset = await TestAsset.GenerateAssetAsync( - AssetName, - NoBannerTestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); + yield return (AssetName, AssetName, NoBannerTestCode.PatchTargetFrameworks(TargetFrameworks.All)); } - public void Dispose() => _testAsset?.Dispose(); - } - - private const string NoBannerTestCode = """ + private const string NoBannerTestCode = """ #file NoBannerTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -153,4 +139,5 @@ public class DummyTestAdapter : ITestFramework public Task ExecuteRequestAsync(ExecuteRequestContext context) => Task.CompletedTask; } """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Properties/launchSettings.json b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Properties/launchSettings.json index ff4d682a58..2610486891 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Properties/launchSettings.json +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "Microsoft.Testing.Platform.Acceptance.IntegrationTests": { "commandName": "Project", - // "commandLineArgs": "--treenode-filter /*/*/[TestClassName]/**", + "commandLineArgs": "--treenode-filter /*/*/*/**", "environmentVariables": { //"TESTINGPLATFORM_HOTRELOAD_ENABLED": "1" } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TelemetryTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TelemetryTests.cs index edb51e17f9..d8e85ba01b 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TelemetryTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TelemetryTests.cs @@ -10,29 +10,28 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class TelemetryTests : BaseAcceptanceTests +public class TelemetryTests : AcceptanceTestBase { private const string AssetName = "TelemetryTest"; - private readonly BuildFixture _buildFixture; + private readonly TestAssetFixture _testAssetFixture; - public TelemetryTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, - BuildFixture buildFixture) - : base(testExecutionContext, acceptanceFixture) + public TelemetryTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _buildFixture = buildFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Telemetry_ByDefault_TelemetryIsEnabled(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic"); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic", disableTelemetry: false); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); string diagContentsPattern = """ @@ -49,18 +48,19 @@ public async Task Telemetry_ByDefault_TelemetryIsEnabled(string tfm) [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Telemetry_WhenOptingOutTelemetry_WithEnvironmentVariable_TelemetryIsDisabled(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( "--diagnostic", new Dictionary() { { EnvironmentVariableConstants.TESTINGPLATFORM_TELEMETRY_OPTOUT, "1" }, - }); + }, + disableTelemetry: false); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); string diagContentsPattern = """ @@ -77,18 +77,19 @@ public async Task Telemetry_WhenOptingOutTelemetry_WithEnvironmentVariable_Telem [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Telemetry_WhenOptingOutTelemetry_With_DOTNET_CLI_EnvironmentVariable_TelemetryIsDisabled(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( "--diagnostic", new Dictionary() { { EnvironmentVariableConstants.DOTNET_CLI_TELEMETRY_OPTOUT, "1" }, - }); + }, + disableTelemetry: false); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); string diagContentsPattern = """ @@ -105,13 +106,13 @@ public async Task Telemetry_WhenOptingOutTelemetry_With_DOTNET_CLI_EnvironmentVa [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Telemetry_WhenEnableTelemetryIsFalse_WithTestApplicationOptions_TelemetryIsDisabled(string tfm) { - string diagPath = Path.Combine(_buildFixture.TargetAssetPathWithDisableTelemetry, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); + string diagPath = Path.Combine(_testAssetFixture.TargetAssetPathWithDisableTelemetry, "bin", "Release", tfm, AggregatedConfiguration.DefaultTestResultFolderName); string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPathWithDisableTelemetry, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic"); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPathWithDisableTelemetry, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--diagnostic", disableTelemetry: false); - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); string diagContentsPattern = """ @@ -127,7 +128,7 @@ public async Task Telemetry_WhenEnableTelemetryIsFalse_WithTestApplicationOption private async Task AssertDiagnosticReportAsync(TestHostResult testHostResult, string diagPathPattern, string diagContentsPattern, string level = "Information", string flushType = "async") { - testHostResult.AssertHasExitCode(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); string outputPattern = $""" Diagnostic file \(level '{level}' with {flushType} flush\): {diagPathPattern} @@ -149,47 +150,33 @@ private async Task AssertDiagnosticReportAsync(TestHostResult testHostRe } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class BuildFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { - private readonly AcceptanceFixture _acceptanceFixture; + private const string WithTelemetry = nameof(WithTelemetry); + private const string WithoutTelemetry = nameof(WithoutTelemetry); - private TestAsset? _testAsset; - private TestAsset? _testAssetWithDisableTelemetry; + public string TargetAssetPath => GetAssetPath(WithTelemetry); - public string TargetAssetPath => _testAsset!.TargetAssetPath; + public string TargetAssetPathWithDisableTelemetry => GetAssetPath(WithoutTelemetry); - public string TargetAssetPathWithDisableTelemetry => _testAssetWithDisableTelemetry!.TargetAssetPath; - - public BuildFixture(AcceptanceFixture acceptanceFixture) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _acceptanceFixture = acceptanceFixture; + yield return (WithTelemetry, AssetName, + TestCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$TelemetryArg$", string.Empty)); + + yield return (WithoutTelemetry, AssetName, + TestCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$TelemetryArg$", ", new TestApplicationOptions() { EnableTelemetry = false }")); } - public async Task InitializeAsync(InitializationContext context) - { - _testAsset = await TestAsset.GenerateAssetAsync( - AssetName, - TestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks()).PatchCodeWithRegularExpression("disableTelemetry", string.Empty)); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - - _testAssetWithDisableTelemetry = await TestAsset.GenerateAssetAsync( - AssetName, - TestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks()).PatchCodeWithRegularExpression("disableTelemetry", ", new TestApplicationOptions() { EnableTelemetry = false }")); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAssetWithDisableTelemetry.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } - - public void Dispose() - { - _testAsset?.Dispose(); - _testAssetWithDisableTelemetry?.Dispose(); - } - } - - private const string TestCode = """ + private const string TestCode = """ #file TelemetryTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -216,7 +203,7 @@ public class Program { public static async Task Main(string[] args) { - ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args disableTelemetry); + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args$TelemetryArg$); builder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_,__) => new DummyTestAdapter()); using ITestApplication app = await builder.BuildAsync(); return await app.RunAsync(); @@ -240,4 +227,5 @@ public class DummyTestAdapter : ITestFramework public Task ExecuteRequestAsync(ExecuteRequestContext context) => Task.CompletedTask; } """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TrxTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TrxTests.cs index 0c48fa907f..d2877d3584 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TrxTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TrxTests.cs @@ -9,23 +9,23 @@ namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class TrxTests : BaseAcceptanceTests +public class TrxTests : AcceptanceTestBase { - private readonly BuildFixture _buildFixture; + private readonly TestAssetFixture _testAssetFixture; - public TrxTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, BuildFixture buildFixture) - : base(testExecutionContext, acceptanceFixture) + public TrxTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _buildFixture = buildFixture; + _testAssetFixture = testAssetFixture; } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Trx_WhenReportTrxIsNotSpecified_TrxReportIsNotGenerated(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, BuildFixture.AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync(); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); string outputPattern = """ Out of process file artifacts produced: @@ -37,26 +37,26 @@ public async Task Trx_WhenReportTrxIsNotSpecified_TrxReportIsNotGenerated(string [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Trx_WhenReportTrxIsSpecified_TrxReportIsGeneratedInDefaultLocation(string tfm) { - string testResultsPath = Path.Combine(_buildFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); + string testResultsPath = Path.Combine(_testAssetFixture.TargetAssetPath, "bin", "Release", tfm, "TestResults"); string trxPathPattern = Path.Combine(testResultsPath, ".*.trx").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, BuildFixture.AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--report-trx"); // number of test is the third param because we have two different test code with different number of tests. - await AssertTrxReportWasGenerated(testHostResult, trxPathPattern, 1); + await AssertTrxReportWasGeneratedAsync(testHostResult, trxPathPattern, 1); } [ArgumentsProvider(nameof(TargetFrameworks.Net), typeof(TargetFrameworks))] public async Task Trx_WhenTestHostCrash_ErrorIsDisplayedInsideTheTrx(string tfm) { string fileName = Guid.NewGuid().ToString("N"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, BuildFixture.AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync( $"--crashdump --report-trx --report-trx-filename {fileName}.trx", new() { { "CRASHPROCESS", "1" } }); - testHostResult.AssertHasExitCode(ExitCodes.TestHostProcessExitedNonGracefully); + testHostResult.AssertExitCodeIs(ExitCodes.TestHostProcessExitedNonGracefully); string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single(); string trxContent = File.ReadAllText(trxFile); @@ -68,10 +68,10 @@ public async Task Trx_WhenTestHostCrash_ErrorIsDisplayedInsideTheTrx(string tfm) public async Task Trx_WhenSkipTest_ItAppearsAsExpectedInsideTheTrx(string tfm) { string fileName = Guid.NewGuid().ToString("N"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPathWithSkippedTest, BuildFixture.AssetNameUsingMSTest, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPathWithSkippedTest, TestAssetFixture.AssetNameUsingMSTest, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {fileName}.trx"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single(); @@ -90,24 +90,24 @@ public async Task Trx_WhenSkipTest_ItAppearsAsExpectedInsideTheTrx(string tfm) [ArgumentsProvider(nameof(TargetFrameworks.Net), typeof(TargetFrameworks))] public async Task Trx_WhenTheTestNameHasInvalidXmlChar_TheTrxCreatedSuccessfully(string tfm) { - string testResultsPath = Path.Combine(_buildFixture.TargetAssetPathWithDataRow, "bin", "Release", tfm, "TestResults"); + string testResultsPath = Path.Combine(_testAssetFixture.TargetAssetPathWithDataRow, "bin", "Release", tfm, "TestResults"); string trxPathPattern = Path.Combine(testResultsPath, ".*.trx").Replace(@"\", @"\\"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPathWithDataRow, BuildFixture.AssetNameUsingMSTest, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPathWithDataRow, TestAssetFixture.AssetNameUsingMSTest, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--report-trx"); // number of test is the third param because we have two different test code with different number of tests. - await AssertTrxReportWasGenerated(testHostResult, trxPathPattern, 2); + await AssertTrxReportWasGeneratedAsync(testHostResult, trxPathPattern, 2); } [ArgumentsProvider(nameof(TargetFrameworks.Net), typeof(TargetFrameworks))] public async Task Trx_UsingDataDriven_CreatesUnitTestTagForEachOneInsideTheTrx(string tfm) { string fileName = Guid.NewGuid().ToString("N"); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPathWithSkippedTest, BuildFixture.AssetNameUsingMSTest, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPathWithSkippedTest, TestAssetFixture.AssetNameUsingMSTest, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {fileName}.trx"); - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single(); @@ -124,113 +124,50 @@ public async Task Trx_UsingDataDriven_CreatesUnitTestTagForEachOneInsideTheTrx(s [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Trx_WhenReportTrxIsSpecifiedWithFullPath_TrxReportShouldFail(string tfm) { - string testResultsPath = Path.Combine(_buildFixture.TargetAssetPath, "aaa", "Release", tfm, "TestResults"); + string testResultsPath = Path.Combine(_testAssetFixture.TargetAssetPath, "aaa", "Release", tfm, "TestResults"); Assert.IsFalse(Directory.Exists(testResultsPath)); - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, BuildFixture.AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {Path.Combine(testResultsPath, "report.trx")}"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); testHostResult.AssertOutputContains("Option '--report-trx-filename' has invalid arguments: file name argument must not contain path (e.g. --report-trx-filename myreport.trx)"); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Trx_WhenReportTrxIsSpecifiedWithRelativePath_TrxReportShouldFail(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, BuildFixture.AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {Path.Combine("aaa", "report.trx")}"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); testHostResult.AssertOutputContains("Option '--report-trx-filename' has invalid arguments: file name argument must not contain path (e.g. --report-trx-filename myreport.trx)"); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Trx_WhenReportTrxIsNotSpecifiedAndReportTrxPathIsSpecified_ErrorIsDisplayed(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, BuildFixture.AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx-filename report.trx"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); testHostResult.AssertOutputContains("Error: '--report-trx-filename' requires '--report-trx' to be enabled"); } [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] public async Task Trx_WhenReportTrxIsSpecifiedAndListTestsIsSpecified_ErrorIsDisplayed(string tfm) { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_buildFixture.TargetAssetPath, BuildFixture.AssetName, tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, TestAssetFixture.AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --list-tests"); - testHostResult.AssertHasExitCode(ExitCodes.InvalidCommandLine); + testHostResult.AssertExitCodeIs(ExitCodes.InvalidCommandLine); testHostResult.AssertOutputContains("Error: '--report-trx' cannot be enabled when using '--list-tests'"); } - [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class BuildFixture : IAsyncInitializable, IDisposable - { - public const string AssetName = "TrxTest"; - public const string AssetNameUsingMSTest = "TrxTestUsingMSTest"; - - private readonly AcceptanceFixture _acceptanceFixture; - - private TestAsset? _testAsset; - private TestAsset? _testAssetWithSkippedTest; - private TestAsset? _testAssetWithDataRow; - - public string TargetAssetPath => _testAsset!.TargetAssetPath; - - public string TargetAssetPathWithSkippedTest => _testAssetWithSkippedTest!.TargetAssetPath; - - public string TargetAssetPathWithDataRow => _testAssetWithDataRow!.TargetAssetPath; - - public BuildFixture(AcceptanceFixture acceptanceFixture) - { - _acceptanceFixture = acceptanceFixture; - } - - public async Task InitializeAsync(InitializationContext context) - { - await Task.WhenAll( - GenerateTrxAsset(), - GenerateMSTest(), - GenerateMSTestWithIgnore()); - - async Task GenerateTrxAsset() - { - _testAsset = await TestAsset.GenerateAssetAsync( - AssetName, - TestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } - - async Task GenerateMSTestWithIgnore() - { - _testAssetWithSkippedTest = await TestAsset.GenerateAssetAsync( - AssetNameUsingMSTest, - MSTestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks()).PatchCodeWithRegularExpression("ignoreTest", "[Ignore]")); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAssetWithSkippedTest.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } - - async Task GenerateMSTest() - { - _testAssetWithDataRow = await TestAsset.GenerateAssetAsync( - AssetNameUsingMSTest, - MSTestCode.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks()).PatchCodeWithRegularExpression("ignoreTest", string.Empty)); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAssetWithDataRow.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); - } - } - - public void Dispose() - { - _testAsset?.Dispose(); - _testAssetWithSkippedTest?.Dispose(); - _testAssetWithDataRow?.Dispose(); - } - } - - private async Task AssertTrxReportWasGenerated(TestHostResult testHostResult, string trxPathPattern, int numberOfTests) + private async Task AssertTrxReportWasGeneratedAsync(TestHostResult testHostResult, string trxPathPattern, int numberOfTests) { - testHostResult.AssertHasExitCode(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.Success); string outputPattern = $""" In process file artifacts produced: @@ -255,11 +192,38 @@ private async Task CheckTrxContentsMatchAsync(string path, string pattern) return Regex.IsMatch(await reader.ReadToEndAsync(), pattern); } - private const string TestCode = """ + [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) + { + public const string AssetName = "TrxTest"; + public const string AssetNameUsingMSTest = "TrxTestUsingMSTest"; + private const string WithSkippedTest = nameof(WithSkippedTest); + private const string WithDataRow = nameof(WithDataRow); + + public string TargetAssetPath => GetAssetPath(AssetName); + + public string TargetAssetPathWithSkippedTest => GetAssetPath(WithSkippedTest); + + public string TargetAssetPathWithDataRow => GetAssetPath(WithDataRow); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + yield return (AssetName, AssetName, TestCode.PatchTargetFrameworks(TargetFrameworks.All)); + yield return (WithSkippedTest, AssetNameUsingMSTest, + MSTestCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$IgnoreTestAttributeOrNothing$", "[Ignore]")); + yield return (WithDataRow, AssetNameUsingMSTest, + MSTestCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$IgnoreTestAttributeOrNothing$", string.Empty)); + } + + private const string TestCode = """ #file TrxTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -305,11 +269,11 @@ public void TestMethod1() global using Microsoft.Testing.Platform.Extensions; """; - private const string MSTestCode = """ + private const string MSTestCode = """ #file TrxTestUsingMSTest.csproj - tfms + $TargetFrameworks$ enable enable Exe @@ -339,7 +303,7 @@ namespace TrxTestUsingMSTest; [TestClass] public class UnitTest1 { - ignoreTest + $IgnoreTestAttributeOrNothing$ [TestMethod] [DataRow("data\0")] [DataRow("data")] @@ -353,4 +317,5 @@ public void TestMethod1(string s) global using Microsoft.Testing.Platform.Extensions; global using Microsoft.VisualStudio.TestTools.UnitTesting; """; + } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/UnhandledExceptionPolicyTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/UnhandledExceptionPolicyTests.cs index ca15ecec8c..dd44227ad5 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/UnhandledExceptionPolicyTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/UnhandledExceptionPolicyTests.cs @@ -3,19 +3,20 @@ using System.Runtime.InteropServices; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; [TestGroup] -public class UnhandledExceptionPolicyTests : BaseAcceptanceTests +public class UnhandledExceptionPolicyTests : AcceptanceTestBase { - private readonly UnhandledExceptionPolicyFixture _unhandledExceptionPolicyFixture; + private readonly TestAssetFixture _testAssetFixture; - public UnhandledExceptionPolicyTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture, UnhandledExceptionPolicyFixture unhandledExceptionPolicyFixture) - : base(testExecutionContext, acceptanceFixture) + public UnhandledExceptionPolicyTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) { - _unhandledExceptionPolicyFixture = unhandledExceptionPolicyFixture; + _testAssetFixture = testAssetFixture; } public enum Mode @@ -44,10 +45,10 @@ public async Task UnhandledExceptionPolicy_ConfigFile_UnobservedTaskException_Sh => await RetryHelper.Retry( async () => { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_unhandledExceptionPolicyFixture.TargetAssetPath, "UnhandledExceptionPolicyTests", tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, "UnhandledExceptionPolicyTests", tfm); using TempDirectory clone = new(); - await clone.CopyDirectoryAsync(testHost.DirectoryName, clone.DirectoryPath, retainAttributes: !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)); - testHost = TestInfrastructure.TestHost.LocateFrom(clone.DirectoryPath, "UnhandledExceptionPolicyTests"); + await clone.CopyDirectoryAsync(testHost.DirectoryName, clone.Path, retainAttributes: !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)); + testHost = TestInfrastructure.TestHost.LocateFrom(clone.Path, "UnhandledExceptionPolicyTests"); string configFileName = Path.Combine(testHost.DirectoryName, "UnhandledExceptionPolicyTests.testingplatformconfig.json"); string contentFile = await File.ReadAllTextAsync(Path.Combine(testHost.DirectoryName, "UnhandledExceptionPolicyTests.testingplatformconfig.json")); @@ -57,40 +58,40 @@ public async Task UnhandledExceptionPolicy_ConfigFile_UnobservedTaskException_Sh case Mode.Enabled: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": true")); testHostResult = await testHost.ExecuteAsync(null, new() { { "UNOBSERVEDTASKEXCEPTION", "1" } }); - Assert.That(testHostResult.StandardOutput.Contains("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException(testhost controller workflow)]", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode != ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); + testHostResult.AssertOutputContains("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException(testhost controller workflow)]"); break; case Mode.Disabled: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": false")); testHostResult = await testHost.ExecuteAsync(null, new() { { "UNOBSERVEDTASKEXCEPTION", "1" } }); - Assert.That(!testHostResult.StandardOutput.Contains("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException]", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode == ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertOutputDoesNotContain("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException]"); break; case Mode.Default: File.Delete(configFileName); testHostResult = await testHost.ExecuteAsync(null, new() { { "UNOBSERVEDTASKEXCEPTION", "1" } }); - Assert.That(!testHostResult.StandardOutput.Contains("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException]", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode == ExitCodes.Success, testHostResult.ToString()); + testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertOutputDoesNotContain("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException]"); break; case Mode.DisabledByEnvironmentVariable: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": true")); testHostResult = await testHost.ExecuteAsync(null, new() - { - { "UNOBSERVEDTASKEXCEPTION", "1" }, - { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "0" }, - }); - Assert.That(!testHostResult.StandardOutput.Contains("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException]", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode == ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + { + { "UNOBSERVEDTASKEXCEPTION", "1" }, + { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "0" }, + }); + testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertOutputDoesNotContain("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException]"); break; case Mode.EnabledByEnvironmentVariable: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": false")); testHostResult = await testHost.ExecuteAsync(null, new() - { - { "UNOBSERVEDTASKEXCEPTION", "1" }, - { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "1" }, - }); - Assert.That(testHostResult.StandardOutput.Contains("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException(testhost controller workflow)]"), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode != ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + { + { "UNOBSERVEDTASKEXCEPTION", "1" }, + { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "1" }, + }); + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); + testHostResult.AssertOutputContains("[UnhandledExceptionHandler.OnTaskSchedulerUnobservedTaskException(testhost controller workflow)]"); break; default: throw new NotImplementedException($"Mode not found '{mode}'"); @@ -104,10 +105,10 @@ public async Task UnhandledExceptionPolicy_EnvironmentVariable_UnhandledExceptio => await RetryHelper.Retry( async () => { - TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_unhandledExceptionPolicyFixture.TargetAssetPath, "UnhandledExceptionPolicyTests", tfm); + TestInfrastructure.TestHost testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, "UnhandledExceptionPolicyTests", tfm); using TempDirectory clone = new(); - await clone.CopyDirectoryAsync(testHost.DirectoryName, clone.DirectoryPath, retainAttributes: !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)); - testHost = TestInfrastructure.TestHost.LocateFrom(clone.DirectoryPath, "UnhandledExceptionPolicyTests"); + await clone.CopyDirectoryAsync(testHost.DirectoryName, clone.Path, retainAttributes: !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)); + testHost = TestInfrastructure.TestHost.LocateFrom(clone.Path, "UnhandledExceptionPolicyTests"); string configFileName = Path.Combine(testHost.DirectoryName, "UnhandledExceptionPolicyTests.testingplatformconfig.json"); string contentFile = await File.ReadAllTextAsync(Path.Combine(testHost.DirectoryName, "UnhandledExceptionPolicyTests.testingplatformconfig.json")); @@ -117,42 +118,42 @@ public async Task UnhandledExceptionPolicy_EnvironmentVariable_UnhandledExceptio case Mode.Enabled: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": true")); testHostResult = await testHost.ExecuteAsync(null, new() { { "UNHANDLEDEXCEPTION", "1" } }); - Assert.That(testHostResult.StandardOutput.Contains("[UnhandledExceptionHandler.OnCurrentDomainUnhandledException(testhost controller workflow)]", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.StandardOutput.Contains("IsTerminating: True", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode != ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + testHostResult.AssertOutputContains("[UnhandledExceptionHandler.OnCurrentDomainUnhandledException(testhost controller workflow)]"); + testHostResult.AssertOutputContains("IsTerminating: True"); + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); break; case Mode.Disabled: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": false")); testHostResult = await testHost.ExecuteAsync(null, new() { { "UNHANDLEDEXCEPTION", "1" } }); Assert.IsTrue(testHostResult.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode != ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); break; case Mode.Default: File.Delete(configFileName); testHostResult = await testHost.ExecuteAsync(null, new() { { "UNHANDLEDEXCEPTION", "1" } }); - Assert.That(testHostResult.ExitCode != ExitCodes.Success, testHostResult.ToString()); + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); break; case Mode.DisabledByEnvironmentVariable: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": true")); testHostResult = await testHost.ExecuteAsync(null, new() - { - { "UNHANDLEDEXCEPTION", "1" }, - { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "0" }, - }); + { + { "UNHANDLEDEXCEPTION", "1" }, + { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "0" }, + }); Assert.IsTrue(testHostResult.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(!testHostResult.StandardOutput.Contains("IsTerminating: True", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode != ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + testHostResult.AssertOutputDoesNotContain("IsTerminating: True"); + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); break; case Mode.EnabledByEnvironmentVariable: File.WriteAllText(configFileName, contentFile.Replace("\"exitProcessOnUnhandledException\": false", "\"exitProcessOnUnhandledException\": false")); testHostResult = await testHost.ExecuteAsync(null, new() - { - { "UNHANDLEDEXCEPTION", "1" }, - { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "1" }, - }); - Assert.That(testHostResult.StandardOutput.Contains("[UnhandledExceptionHandler.OnCurrentDomainUnhandledException(testhost controller workflow)]", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.StandardOutput.Contains("IsTerminating: True", StringComparison.OrdinalIgnoreCase), testHostResult.ToString()); - Assert.That(testHostResult.ExitCode != ExitCodes.Success, $"{testHostResult}\n{File.ReadAllText(configFileName)}"); + { + { "UNHANDLEDEXCEPTION", "1" }, + { EnvironmentVariableConstants.TESTINGPLATFORM_EXIT_PROCESS_ON_UNHANDLED_EXCEPTION, "1" }, + }); + testHostResult.AssertOutputContains("[UnhandledExceptionHandler.OnCurrentDomainUnhandledException(testhost controller workflow)]"); + testHostResult.AssertOutputContains("IsTerminating: True"); + testHostResult.AssertExitCodeIsNot(ExitCodes.Success); break; default: throw new NotImplementedException($"Mode not found '{mode}'"); @@ -162,33 +163,23 @@ public async Task UnhandledExceptionPolicy_EnvironmentVariable_UnhandledExceptio }, 3, TimeSpan.FromSeconds(3)); [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] - public sealed class UnhandledExceptionPolicyFixture : IAsyncInitializable, IDisposable + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) { - private readonly AcceptanceFixture _acceptanceFixture; - private TestAsset? _testAsset; - - public UnhandledExceptionPolicyFixture(AcceptanceFixture acceptanceFixture) - { - _acceptanceFixture = acceptanceFixture; - } + private const string AssetName = "UnhandledExceptionPolicyTests"; - public string TargetAssetPath => _testAsset!.TargetAssetPath; + public string TargetAssetPath => GetAssetPath(AssetName); - public async Task InitializeAsync(InitializationContext context) + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - _testAsset = await TestAsset.GenerateAssetAsync("UnhandledExceptionPolicyTests", Sources.PatchCodeWithRegularExpression("tfms", TargetFrameworks.All.ToMSBuildTargetFrameworks())); - await DotnetCli.RunAsync($"build -nodeReuse:false {_testAsset.TargetAssetPath} -c Release", _acceptanceFixture.NuGetGlobalPackagesFolder); + yield return (AssetName, AssetName, Sources.PatchTargetFrameworks(TargetFrameworks.All)); } - public void Dispose() => _testAsset?.Dispose(); - } - - private const string Sources = """ + private const string Sources = """ #file UnhandledExceptionPolicyTests.csproj - tfms + $TargetFrameworks$ Exe true enable @@ -309,4 +300,5 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context) } } """; + } } diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Logging/FileLoggerTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Logging/FileLoggerTests.cs index 8f767ac824..66e56abca5 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Logging/FileLoggerTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Logging/FileLoggerTests.cs @@ -19,7 +19,7 @@ public FileLoggerTests(ITestExecutionContext testExecutionContext) public void Write_IfMalformedUTF8_ShouldNotCrash() { using TempDirectory tempDirectory = new(nameof(Write_IfMalformedUTF8_ShouldNotCrash)); - using FileLogger fileLogger = new(tempDirectory.DirectoryPath, fileName: null, LogLevel.Trace, "Test", true, new SystemClock(), new SystemTask(), new SystemConsole()); + using FileLogger fileLogger = new(tempDirectory.Path, fileName: null, LogLevel.Trace, "Test", true, new SystemClock(), new SystemTask(), new SystemConsole()); fileLogger.Log(LogLevel.Trace, "\uD886", null, LoggingExtensions.Formatter, "Category"); } } diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/SourceCodeExtensions.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/SourceCodeExtensions.cs index a3c70e3e5e..5a5cc262dc 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/SourceCodeExtensions.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/SourceCodeExtensions.cs @@ -1,12 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Text.RegularExpressions; +using Microsoft.Testing.Framework; namespace Microsoft.Testing.TestInfrastructure; public static class SourceCodeExtensions { - public static string PatchCodeWithRegularExpression(this string code, string pattern, string value) - => Regex.Replace(code, pattern, value); + public static string PatchCodeWithReplace(this string code, string pattern, string value) + => code.Replace(pattern, value); + + public static string PatchTargetFrameworks(this string code, TestArgumentsEntry[] targetFrameworks) + => PatchCodeWithReplace(code, "$TargetFrameworks$", targetFrameworks.ToMSBuildTargetFrameworks()); } diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs index 729c34f3cf..957ca7161f 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs @@ -21,11 +21,11 @@ public TempDirectory(string? subDirectory = null, bool arcadeConvention = true, cleanup = false; } - (_baseDirectory, DirectoryPath) = CreateUniqueDirectory(subDirectory, arcadeConvention); + (_baseDirectory, Path) = CreateUniqueDirectory(subDirectory, arcadeConvention); _cleanup = cleanup; } - public string DirectoryPath { get; } + public string Path { get; } public void Dispose() { @@ -49,12 +49,12 @@ protected virtual void Dispose(bool disposing) } public DirectoryInfo CreateDirectory(string dir) - => Directory.CreateDirectory(Path.Combine(DirectoryPath, dir)); + => Directory.CreateDirectory(System.IO.Path.Combine(Path, dir)); public static async Task WriteFileAsync(string targetDirectory, string fileName, string fileContents) { - string finalFile = Path.Combine(targetDirectory, fileName); - Directory.CreateDirectory(Path.GetDirectoryName(finalFile)!); + string finalFile = System.IO.Path.Combine(targetDirectory, fileName); + Directory.CreateDirectory(System.IO.Path.GetDirectoryName(finalFile)!); using var fs = new FileStream(finalFile, FileMode.CreateNew); using var stream = new StreamWriter(fs); await stream.WriteLineAsync(fileContents); @@ -72,13 +72,13 @@ public static async Task CopyDirectoryAsync(DirectoryInfo source, DirectoryInfo { if (retainAttributes) { - File.Copy(fi.FullName, Path.Combine(target.FullName, fi.Name)); + File.Copy(fi.FullName, System.IO.Path.Combine(target.FullName, fi.Name)); } else { using FileStream fileStream = File.OpenRead(fi.FullName); using var destinationStream = new FileStream( - Path.Combine(target.FullName, fi.Name), + System.IO.Path.Combine(target.FullName, fi.Name), FileMode.CreateNew); await fileStream.CopyToAsync(destinationStream); } @@ -102,7 +102,7 @@ public static void CopyDirectory(DirectoryInfo source, DirectoryInfo target) // Copy each file into the new directory. foreach (FileInfo fi in source.GetFiles()) { - fi.CopyTo(Path.Combine(target.FullName, fi.Name), true); + fi.CopyTo(System.IO.Path.Combine(target.FullName, fi.Name), true); } // Copy each subdirectory using recursion. @@ -121,7 +121,7 @@ public string[] CopyFile(params string[] filePaths) var paths = new List(filePaths.Length); foreach (string filePath in filePaths) { - string destination = Path.Combine(DirectoryPath, Path.GetFileName(filePath)); + string destination = System.IO.Path.Combine(Path, System.IO.Path.GetFileName(filePath)); File.Copy(filePath, destination); paths.Add(destination); } @@ -134,7 +134,7 @@ public string[] CopyFile(params string[] filePaths) /// public string CopyFile(string filePath) { - string destination = Path.Combine(DirectoryPath, Path.GetFileName(filePath)); + string destination = System.IO.Path.Combine(Path, System.IO.Path.GetFileName(filePath)); File.Copy(filePath, destination); return destination; } @@ -150,9 +150,9 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect if (arcadeConvention) { string currentDirectory = AppContext.BaseDirectory; - while (Path.GetFileName(currentDirectory) != "artifacts" && currentDirectory is not null) + while (System.IO.Path.GetFileName(currentDirectory) != "artifacts" && currentDirectory is not null) { - currentDirectory = Path.GetDirectoryName(currentDirectory)!; + currentDirectory = System.IO.Path.GetDirectoryName(currentDirectory)!; } if (currentDirectory is null) @@ -160,16 +160,16 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect throw new InvalidOperationException("artifacts folder not found"); } - string directoryPath = Path.Combine(currentDirectory, "tmp", Constants.BuildConfiguration, "testsuite", RandomId.Next()); + string directoryPath = System.IO.Path.Combine(currentDirectory, "tmp", Constants.BuildConfiguration, "testsuite", RandomId.Next()); Directory.CreateDirectory(directoryPath); - string directoryProp = Path.Combine(directoryPath, "Directory.Build.props"); - string directoryTarget = Path.Combine(directoryPath, "Directory.Build.targets"); + string directoryProp = System.IO.Path.Combine(directoryPath, "Directory.Build.props"); + string directoryTarget = System.IO.Path.Combine(directoryPath, "Directory.Build.targets"); File.WriteAllText(directoryProp, $""" - {Path.GetDirectoryName(currentDirectory)}/ + {System.IO.Path.GetDirectoryName(currentDirectory)}/ """); @@ -181,7 +181,7 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect string finalDirectory = directoryPath; if (!string.IsNullOrWhiteSpace(subDirectory)) { - finalDirectory = Path.Combine(directoryPath, subDirectory); + finalDirectory = System.IO.Path.Combine(directoryPath, subDirectory); } Directory.CreateDirectory(finalDirectory); @@ -191,11 +191,11 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect else { string temp = GetTempPath(); - string directoryPath = Path.Combine(temp, "testingplatform", RandomId.Next()); + string directoryPath = System.IO.Path.Combine(temp, "testingplatform", RandomId.Next()); string finalDirectory = directoryPath; if (!string.IsNullOrWhiteSpace(subDirectory)) { - finalDirectory = Path.Combine(directoryPath, subDirectory); + finalDirectory = System.IO.Path.Combine(directoryPath, subDirectory); } Directory.CreateDirectory(finalDirectory); @@ -212,7 +212,7 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect // place where we are allowed to use it. All other methods should use our GetTempPath (this method). private static string GetTempPath() => Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") - ?? Path.GetTempPath(); + ?? System.IO.Path.GetTempPath(); public static void TryRemoveDirectory(string directory) { @@ -233,11 +233,11 @@ public void Add(string fileContents) List files = InlineFileParser.ParseFiles(fileContents); foreach (InlineFile file in files) { - File.WriteAllText(Path.Combine(DirectoryPath, file.Name), file.Content, Encoding.UTF8); + File.WriteAllText(System.IO.Path.Combine(Path, file.Name), file.Content, Encoding.UTF8); } } - public override string ToString() => DirectoryPath; + public override string ToString() => Path; internal sealed class InlineFile(string name, string content) { diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAsset.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAsset.cs index 2fbb394640..9a9e3fa8e5 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAsset.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAsset.cs @@ -39,7 +39,7 @@ protected virtual void Dispose(bool disposing) _isDisposed = true; } - public string TargetAssetPath => _tempDirectory.DirectoryPath; + public string TargetAssetPath => _tempDirectory.Path; private static (string Name, string Content) ParseFile(string fileContent) { @@ -86,7 +86,7 @@ public static async Task GenerateAssetAsync(string assetName, string foreach (string fileContent in splitFiles) { (string, string) fileInfo = ParseFile(fileContent); - await TempDirectory.WriteFileAsync(testAsset._tempDirectory.DirectoryPath, fileInfo.Item1, fileInfo.Item2); + await TempDirectory.WriteFileAsync(testAsset._tempDirectory.Path, fileInfo.Item1, fileInfo.Item2); } return testAsset; diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAssetFixtureBase.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAssetFixtureBase.cs new file mode 100644 index 0000000000..e92df97cfb --- /dev/null +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAssetFixtureBase.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; + +using Microsoft.Testing.Framework; + +namespace Microsoft.Testing.TestInfrastructure; + +public abstract class TestAssetFixtureBase : IDisposable, IAsyncInitializable +{ + private readonly ConcurrentDictionary _testAssets = new(); + private readonly TempDirectory _nugetGlobalPackagesDirectory; + private bool _disposedValue; + + protected TestAssetFixtureBase(TempDirectory nugetGlobalPackagesDirectory) + { + _nugetGlobalPackagesDirectory = nugetGlobalPackagesDirectory; + } + + public string GetAssetPath(string assetID) + => !_testAssets.TryGetValue(assetID, out TestAsset? value) + ? throw new ArgumentNullException(nameof(assetID), $"Cannot find target path for test asset '{assetID}'") + : value.TargetAssetPath; + + public async Task InitializeAsync(InitializationContext context) +#if NET + => await Parallel.ForEachAsync(GetAssetsToGenerate(), async (asset, _) => + { + var testAsset = await TestAsset.GenerateAssetAsync(asset.Name, asset.Code); + await DotnetCli.RunAsync($"build -nodeReuse:false {testAsset.TargetAssetPath} -c Release", _nugetGlobalPackagesDirectory.Path); + _testAssets.TryAdd(asset.ID, testAsset); + }); +#else + => await Task.WhenAll(GetAssetsToGenerate().Select(async asset => + { + var testAsset = await TestAsset.GenerateAssetAsync(asset.Name, asset.Code); + await DotnetCli.RunAsync($"build -nodeReuse:false {testAsset.TargetAssetPath} -c Release", _nugetGlobalPackagesDirectory.Path); + _testAssets.TryAdd(asset.ID, testAsset); + })); +#endif + + public abstract IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate(); + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + Parallel.ForEach(_testAssets, (assetPair, _) => assetPair.Value.Dispose()); + } + + _disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } +} diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs index 270d6cd567..6c51549290 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestHost.cs @@ -21,7 +21,7 @@ private TestHost(string testHostFullName, string testHostModuleName) public string DirectoryName { get; } - public async Task ExecuteAsync(string? command = null, Dictionary? environmentVariables = null) + public async Task ExecuteAsync(string? command = null, Dictionary? environmentVariables = null, bool disableTelemetry = true) { if (command?.StartsWith(_testHostModuleName, StringComparison.OrdinalIgnoreCase) ?? false) { @@ -29,6 +29,12 @@ public async Task ExecuteAsync(string? command = null, Dictionar } environmentVariables ??= new Dictionary(); + + if (disableTelemetry) + { + environmentVariables.Add("DOTNET_CLI_TELEMETRY_OPTOUT", "1"); + } + foreach (DictionaryEntry entry in Environment.GetEnvironmentVariables()) { // Skip all unwanted environment variables.