diff --git a/.gitignore b/.gitignore index d13211f267..0d22715429 100644 --- a/.gitignore +++ b/.gitignore @@ -104,14 +104,14 @@ ehthumbs.db # =========================== # Custom ignores # =========================== -packages -artifacts -[tT]ools +/packages/ +/artifacts/ +/[tT]ools/ *.dmp *.nupkg *.zip -src/package/sign/sign.nuget.targets +/src/package/sign/sign.nuget.targets # =========================== # Localized resx files @@ -140,5 +140,5 @@ src/package/sign/sign.nuget.targets # =========================== logs/ -.fake -.ionide +.fake/ +.ionide/ diff --git a/TestPlatform.sln b/TestPlatform.sln index 05b0609b3c..f4b952a31f 100644 --- a/TestPlatform.sln +++ b/TestPlatform.sln @@ -181,21 +181,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataColl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vstest.ProgrammerTests", "test\vstest.ProgrammerTests\vstest.ProgrammerTests.csproj", "{B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testhost.arm64", "src\testhost.arm64\testhost.arm64.csproj", "{186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}" EndProject Global - GlobalSection(SharedMSBuildProjectFiles) = preSolution - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13 - EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 @@ -955,7 +947,7 @@ Global {61F7F446-9EF3-4768-B33A-4D75F60E1059} = {7D4082EA-7AC9-4DFB-98E8-C5E08BDC0EC3} {D5296435-3A3F-4B1A-81D1-434EC9E97DEF} = {5E7F18A8-F843-4C8A-AB02-4C7D9205C6CF} {790B8030-00C2-4121-B125-EDC4CE329BA3} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} - {27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} + {27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {D9A30E32-D466-4EC5-B4F2-62E17562279B} {71CB42FF-E750-4A3B-9C3A-AC938853CC89} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} {10B6ADE1-F808-4612-801D-4452F5B52242} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} {46250E12-4CF1-4051-B4A7-80C8C06E0068} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} @@ -1020,4 +1012,12 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD} EndGlobalSection + GlobalSection(SharedMSBuildProjectFiles) = preSolution + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13 + EndGlobalSection EndGlobal diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b2d56f3b6e..ed76285bba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,6 +8,9 @@ jobs: variables: buildConfiguration: 'Release' steps: + - checkout: self + fetchDepth: 1 + clean: true - task: BatchScript@1 displayName: 'Run script build.cmd' inputs: @@ -57,12 +60,27 @@ jobs: variables: buildConfiguration: 'Release' steps: + - checkout: self + fetchDepth: 1 + clean: true + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' artifactName: 'testArtifacts' targetPath: '$(Build.SourcesDirectory)\artifacts' + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: PowerShell@2 inputs: targetType: 'inline' @@ -84,6 +102,12 @@ jobs: targetType: 'inline' script: 'ls "$(Build.SourcesDirectory)\artifacts\Release\packages"' + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: PowerShell@2 displayName: 'Disable Strong Name Validation' inputs: @@ -153,6 +177,9 @@ jobs: buildConfiguration: 'Release' VSTestRTMBuild: 'false' steps: + - checkout: self + fetchDepth: 1 + clean: true - script: chmod +x ./scripts/vsts-prebuild.sh displayName: 'Preparing for set version' diff --git a/build.cmd b/build.cmd index f2a9a38318..51620a6421 100644 --- a/build.cmd +++ b/build.cmd @@ -3,4 +3,5 @@ REM Copyright (c) Microsoft. All rights reserved. powershell -ExecutionPolicy Bypass -NoProfile -NoLogo -Command "%~dp0scripts\build.ps1 %*; exit $LastExitCode;" + if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 507be1ba0f..cff737dc9c 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -3,47 +3,47 @@ [CmdletBinding()] Param( - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [ValidateSet("Debug", "Release")] [Alias("c")] [System.String] $Configuration = "Debug", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("r")] [System.String] $TargetRuntime = "win7-x64", # Versioning scheme = Major(15).Minor(RTW, Updates).SubUpdates(preview4, preview5, RC etc) # E.g. VS 2017 Update 1 Preview will have version 15.1.1 - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("v")] [System.String] $Version, # Will set this later by reading TestPlatform.Settings.targets file. - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("vs")] [System.String] $VersionSuffix = "dev", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("bn")] [System.String] $BuildNumber = "20991231-99", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("ff")] [System.Boolean] $FailFast = $true, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("noloc")] [Switch] $DisableLocalizedBuild, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("ci")] [Switch] $CIBuild, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("pt")] [Switch] $PublishTestArtifacts, # Build specific projects - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("p")] [System.String[]] $ProjectNamePatterns = @(), @@ -61,11 +61,10 @@ $ErrorView = 'Normal' # Set Version from scripts/build/TestPlatform.Settings.targets, when we are running locally and not providing the version as the parameter # or when the build is done directly in VS -if([string]::IsNullOrWhiteSpace($Version)) -{ +if ([string]::IsNullOrWhiteSpace($Version)) { $Version = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Settings.targets)).Project.PropertyGroup[0].TPVersionPrefix | - ForEach-Object { $_.Trim() } | - Select-Object -First 1 + ForEach-Object { $_.Trim() } | + Select-Object -First 1 Write-Verbose "Version was not provided using version '$Version' from TestPlatform.Settings.targets" } @@ -128,21 +127,18 @@ if ($env:PATH -notlike "*$attachVsPath") { # so nuget restores it correctly. $vsSdkBuildToolsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props)).Project.PropertyGroup.VSSdkBuildToolsVersion $vsixUtilDir = "$env:TP_ROOT_DIR\packages\microsoft.vssdk.buildtools" -if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) -{ +if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) { Remove-Item -Recurse -Force $vsixUtilDir } # Procdump gets regularly eaten by antivirus or something. Remove the package dir if it gets broken # so nuget restores it correctly. $procdumpDir = "$env:TP_ROOT_DIR\packages\procdump" -if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) -{ +if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) { Remove-Item -Recurse -Force $procdumpDir } -function Invoke-Build -{ +function Invoke-Build { $timer = Start-Timer Write-Log "Invoke-Build: Start build." $dotnetExe = Get-DotNetPath @@ -157,8 +153,7 @@ function Invoke-Build Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}" } -function Invoke-TestAssetsBuild -{ +function Invoke-TestAssetsBuild { $timer = Start-Timer Write-Log "Invoke-TestAssetsBuild: Start test assets build." $dotnetExe = Get-DotNetPath @@ -170,6 +165,94 @@ function Invoke-TestAssetsBuild Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- add NuGet source" Invoke-Exe -IgnoreExitCode 1 $nugetExe -Arguments "sources add -Name ""locally-built-testplatform-packages"" -Source $env:TP_TESTARTIFACTS\packages\ -ConfigFile ""$nugetConfig""" Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" + + # Compatibility matrix build. + $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" + $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) + + # Restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll). + # These properties are coming from TestPlatform.Dependencies.props. + $vstestConsoleVersionProperties = @( + "VSTestConsoleLatestVersion" + "VSTestConsoleLatestPreviewVersion" + "VSTestConsoleLatestStableVersion" + "VSTestConsoleRecentStableVersion" + "VSTestConsoleMostDownloadedVersion" + "VSTestConsolePreviousStableVersion" + "VSTestConsoleLegacyStableVersion" + ) + + foreach ($propertyName in $vstestConsoleVersionProperties) { + if ("VSTestConsoleLatestVersion" -eq $propertyName) { + # NETTestSdkVersion has the version of the locally built package. + $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" + } + else { + $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + } + + # The command line tool does not like the package ranges. + $vsTestConsoleVersion = $vsTestConsoleVersion -replace "(\[|\])" + if (-not $vsTestConsoleVersion) { + throw "VSTestConsoleVersion for $propertyName is empty." + } + + # We don't use the results of this build anywhere, we just use them to restore the packages to nuget cache + # because using nuget.exe install errors out in various weird ways. + Invoke-Exe $dotnetExe -Arguments "build $env:TP_ROOT_DIR\test\TestAssets\Tools\Tools.csproj --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$vsTestConsoleVersion" + } + + # Build with multiple versions of MSTest. The projects are directly in the root. + # The folder structure in VS is not echoed in the TestAssets directory. + $projects = @( + "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject1\MSTestProject1.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject2\MSTestProject2.csproj" + # Don't use this one, it does not use the variables for mstest and test sdk. + # "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" + ) + + $msTestVersionProperties = @( + "MSTestFrameworkLatestPreviewVersion" + "MSTestFrameworkLatestStableVersion" + "MSTestFrameworkRecentStableVersion" + "MSTestFrameworkMostDownloadedVersion" + "MSTestFrameworkPreviousStableVersion" + "MSTestFrameworkLegacyStableVersion" + ) + + foreach ($project in $projects) { + # We use the same version properties for NET.Test.Sdk as for VSTestConsole, for now. + foreach ($sdkPropertyName in $vstestConsoleVersionProperties) { + if ("VSTestConsoleLatestVersion" -eq $sdkPropertyName) { + # NETTestSdkVersion has the version of the locally built package. + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" + } + else { + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup.$sdkPropertyName + } + + if (-not $netTestSdkVersion) { + throw "NetTestSdkVersion for $sdkPropertyName is empty." + } + + $dirNetTestSdkVersion = $netTestSdkVersion -replace "\[|\]" + $dirNetTestSdkPropertyName = $sdkPropertyName -replace "Framework" -replace "Version" -replace "VSTestConsole", "NETTestSdk" + + foreach ($propertyName in $msTestVersionProperties) { + $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + + if (-not $mstestVersion) { + throw "MSTestVersion for $propertyName is empty." + } + + $dirMSTestVersion = $mstestVersion -replace "\[|\]" + $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" + } + } + } + + # end } finally { Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- remove NuGet source" @@ -208,8 +291,7 @@ function Publish-PatchedDotnet { Copy-Item $buildArtifactsPath $dotnetTestArtifactsSdkPath -Force } -function Publish-Package -{ +function Publish-Package { $timer = Start-Timer Write-Log "Publish-Package: Started." $fullCLRPackage451Dir = Get-FullCLRPackageDirectory @@ -358,73 +440,73 @@ function Publish-Package # into the output folder (for some reason), and we overwrite it with actual uap10.0 version below Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.ObjectModel\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkCore10 = $coreCLR10PackageDir # netcoreapp1.0 - $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkCore10 = $coreCLR10PackageDir # netcoreapp1.0 + $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.ObjectModel\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.PlatformAbstractions Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.PlatformAbstractions\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.PlatformAbstractions\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.CoreUtilities Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CoreUtilities\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CoreUtilities\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.AdapterUtilities Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.AdapterUtilities\bin\$TPB_Configuration") ` - -files @{ - # "net20" = $net20PackageDir # net20 - "net45/any" = $net45PackageDir # $net4 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + # "net20" = $net20PackageDir # net20 + "net45/any" = $net45PackageDir # $net4 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } ################################################################################ # Publish Microsoft.TestPlatform.CrossPlatEngine Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CrossPlatEngine\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - } + -files @{ + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + } ################################################################################ # Publish msdia @@ -455,7 +537,7 @@ function Publish-Package "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.dll", "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.pdb" ) - foreach($file in $loggers) { + foreach ($file in $loggers) { Write-Verbose "Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force" Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force @@ -464,7 +546,7 @@ function Publish-Package } # Move logger resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Move-Loc-Files $fullCLRPackage451Dir $fullCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll" Move-Loc-Files $coreCLR20PackageDir $coreCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll" Move-Loc-Files $fullCLRPackage451Dir $fullCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.resources.dll" @@ -505,7 +587,7 @@ function Publish-Package # Copy-Item $blameDataCollectorNetStandard\procdump $coreCLRExtensionsDir\procdump -Force # Copy blame data collector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $blameDataCollectorNetFull $fullCLRExtensionsDir "Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll" Copy-Loc-Files $blameDataCollectorNetStandard $coreCLRExtensionsDir "Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll" } @@ -519,7 +601,7 @@ function Publish-Package Copy-Item $eventLogDataCollectorNetFull\Microsoft.TestPlatform.Extensions.EventLogCollector.pdb $coreCLRExtensionsDir -Force # Copy EventLogCollector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $eventLogDataCollectorNetFull $fullCLRExtensionsDir "Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll" Copy-Loc-Files $eventLogDataCollectorNetFull $coreCLRExtensionsDir "Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll" } @@ -528,13 +610,13 @@ function Publish-Package $codeCoverageExternalsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.MicrosoftInternalCodeCoverageVersion $codeCoverageIOPackagesDir = Join-Path $env:TP_PACKAGES_DIR "microsoft.visualstudio.coverage.io\$codeCoverageExternalsVersion\lib\$TPB_TargetFrameworkStandard" Copy-Item $codeCoverageIOPackagesDir\Microsoft.VisualStudio.Coverage.IO.dll $coreCLR20PackageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $codeCoverageIOPackagesDir $coreCLR20PackageDir "Microsoft.VisualStudio.Coverage.IO.resources.dll" } # If there are some dependencies for the TestHostRuntimeProvider assemblies, those need to be moved too. $runtimeproviders = @("Microsoft.TestPlatform.TestHostRuntimeProvider.dll", "Microsoft.TestPlatform.TestHostRuntimeProvider.pdb") - foreach($file in $runtimeproviders) { + foreach ($file in $runtimeproviders) { Write-Verbose "Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force" Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force @@ -573,8 +655,8 @@ function Publish-Package Copy-Item $testhostCore20PackageDir\testhost.pdb $coreCLR20PackageDir -Force Get-Item "$testhostCore20PackageDir\*" | - Where-Object { $_.Name -notin ("x64", "x86", "win7-x64", "win7-x86", "testhost.deps.json", "testhost.runtimeconfig.json")} | - Copy-Item -Recurse -Destination $fullCLRTestHostDir -Force + Where-Object { $_.Name -notin ("x64", "x86", "win7-x64", "win7-x86", "testhost.deps.json", "testhost.runtimeconfig.json") } | + Copy-Item -Recurse -Destination $fullCLRTestHostDir -Force Copy-Item $standaloneTesthost $fullCLRTestHostDir -Force # For libraries that are externally published, copy the output into artifacts. These will be signed and packaged independently. @@ -620,10 +702,8 @@ function Publish-Package Write-Log "Publish-Package: Complete. {$(Get-ElapsedTime($timer))}" } -function Publish-Tests -{ - if($TPB_PublishTests) - { +function Publish-Tests { + if ($TPB_PublishTests) { Write-Log "Publish-Tests: Started." # Adding only Perf project for now @@ -647,21 +727,18 @@ function Publish-Tests } } -function Publish-PackageInternal($packagename, $framework, $output) -{ +function Publish-PackageInternal($packagename, $framework, $output) { $dotnetExe = Get-DotNetPath Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } -function Publish-PackageWithRuntimeInternal($packagename, $framework, $runtime, $selfcontained, $output) -{ +function Publish-PackageWithRuntimeInternal($packagename, $framework, $runtime, $selfcontained, $output) { $dotnetExe = Get-DotNetPath Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --runtime $runtime --self-contained $selfcontained --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } -function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) -{ - foreach($lang in $language) { +function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) { + foreach ($lang in $language) { $dllToCopy = Join-Path $sourceDir\$lang $dllName $destinationFolder = Join-Path $destinationDir $lang if (-not (Test-Path $destinationFolder)) { @@ -671,9 +748,8 @@ function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) } } -function Move-Loc-Files($sourceDir, $destinationDir, $dllName) -{ - foreach($lang in $language) { +function Move-Loc-Files($sourceDir, $destinationDir, $dllName) { + foreach ($lang in $language) { $dllToCopy = Join-Path $sourceDir\$lang $dllName $destinationFolder = Join-Path $destinationDir $lang if (-not (Test-Path $destinationFolder)) { @@ -683,8 +759,7 @@ function Move-Loc-Files($sourceDir, $destinationDir, $dllName) } } -function Create-VsixPackage -{ +function Create-VsixPackage { Write-Log "Create-VsixPackage: Started." $timer = Start-Timer @@ -703,7 +778,7 @@ function Create-VsixPackage # Copy Microsoft.VisualStudio.TraceDataCollector to Extensions $traceDataCollectorPackageDirectory = Join-Path $env:TP_PACKAGES_DIR "Microsoft.VisualStudio.TraceDataCollector\$codeCoverageExternalsVersion\lib\$TPB_TargetFramework472" Copy-Item $traceDataCollectorPackageDirectory\Microsoft.VisualStudio.TraceDataCollector.dll $extensionsPackageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $traceDataCollectorPackageDirectory $extensionsPackageDir "Microsoft.VisualStudio.TraceDataCollector.resources.dll" } @@ -724,7 +799,7 @@ function Create-VsixPackage # Copy Microsoft.VisualStudio.IO to root $codeCoverageIOPackageDirectory = Join-Path $env:TP_PACKAGES_DIR "Microsoft.VisualStudio.Coverage.IO\$codeCoverageExternalsVersion\lib\$TPB_TargetFramework451" Copy-Item $codeCoverageIOPackageDirectory\Microsoft.VisualStudio.Coverage.IO.dll $packageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $codeCoverageIOPackageDirectory $packageDir "Microsoft.VisualStudio.Coverage.IO.resources.dll" } @@ -777,8 +852,7 @@ function Create-VsixPackage $msbuildPath = Locate-MSBuildPath # Create vsix only when msbuild is installed. - if(![string]::IsNullOrEmpty($msbuildPath)) - { + if (![string]::IsNullOrEmpty($msbuildPath)) { # Copy the vsix project to artifacts directory to modify manifest New-Item $vsixProjectDir -Type Directory -Force Copy-Item -Recurse $vsixSourceDir\* $vsixProjectDir -Force @@ -789,16 +863,14 @@ function Create-VsixPackage # Build vsix project to get TestPlatform.vsix Invoke-Exe $msbuildPath -Arguments """$vsixProjectDir\TestPlatform.csproj"" -p:Configuration=$Configuration" } - else - { + else { throw ".. Create-VsixPackage: Cannot generate vsix as msbuild.exe not found at '$msbuildPath'." } Write-Log "Create-VsixPackage: Complete. {$(Get-ElapsedTime($timer))}" } -function Create-NugetPackages -{ +function Create-NugetPackages { $timer = Start-Timer Write-Log "Create-NugetPackages: Started." @@ -807,9 +879,14 @@ function Create-NugetPackages Copy-Item (Join-Path $env:TP_PACKAGE_PROJ_DIR "Icon.png") $stagingDir -Force + # Remove all locally built nuget packages before we start creating them + # we are leaving them in the folder after uzipping them for easier review. + if (Test-Path $packageOutputDir) { + Remove-Item $packageOutputDir -Recurse -Force + } if (-not (Test-Path $packageOutputDir)) { - New-Item $packageOutputDir -type directory -Force + New-Item $packageOutputDir -Type directory -Force } $tpNuspecDir = Join-Path $env:TP_PACKAGE_PROJ_DIR "nuspec" @@ -927,8 +1004,7 @@ function Create-NugetPackages Write-Log "Create-NugetPackages: Complete. {$(Get-ElapsedTime($timer))}" } -function Copy-CodeCoverage-Package-Artifacts -{ +function Copy-CodeCoverage-Package-Artifacts { # Copy TraceDataCollector to Microsoft.CodeCoverage folder. $codeCoverageExternalsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.MicrosoftInternalCodeCoverageVersion $traceDataCollectorPackagesDir = Join-Path $env:TP_PACKAGES_DIR "microsoft.visualstudio.tracedatacollector\$codeCoverageExternalsVersion\lib\$TPB_TargetFrameworkNS20" @@ -963,13 +1039,12 @@ function Copy-CodeCoverage-Package-Artifacts Copy-Item $codeCoverageImMacosPackagesDir\x64 $microsoftCodeCoveragePackageDir\InstrumentationEngine\macos\ -Force -Recurse # Copy TraceDataCollector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $traceDataCollectorPackagesDir $microsoftCodeCoveragePackageDir "Microsoft.VisualStudio.TraceDataCollector.resources.dll" } } -function Copy-PackageItems($packageName) -{ +function Copy-PackageItems($packageName) { # Packages published separately are copied into their own artifacts directory # E.g. src\Microsoft.TestPlatform.ObjectModel\bin\Debug\net451\* is copied # to artifacts\Debug\Microsoft.TestPlatform.ObjectModel\net451 @@ -984,8 +1059,7 @@ function Copy-PackageItems($packageName) Copy-Item -Path $binariesDirectory -Destination $publishDirectory -Recurse -Force } -function Update-LocalizedResources -{ +function Update-LocalizedResources { $timer = Start-Timer $dotnetExe = Get-DotNetPath @@ -1003,8 +1077,7 @@ function Update-LocalizedResources # # Helper functions # -function Get-DotNetPath -{ +function Get-DotNetPath { $dotnetPath = Join-Path $env:TP_TOOLS_DIR "dotnet\dotnet.exe" if (-not (Test-Path $dotnetPath)) { Write-Error "Dotnet.exe not found at $dotnetPath. Did the dotnet cli installation succeed?" @@ -1013,33 +1086,27 @@ function Get-DotNetPath return $dotnetPath } -function Get-FullCLRPackageDirectory -{ +function Get-FullCLRPackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFramework451\$TPB_TargetRuntime") } -function Get-FullCLRPackageDirectory45 -{ +function Get-FullCLRPackageDirectory45 { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFramework45\$TPB_TargetRuntime") } -function Get-CoreCLR20PackageDirectory -{ +function Get-CoreCLR20PackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore20") } -function Get-CoreCLR10PackageDirectory -{ +function Get-CoreCLR10PackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore10") } -function Get-CoreCLR20TestHostPackageDirectory -{ +function Get-CoreCLR20TestHostPackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore20\TestHost") } -function Locate-MSBuildPath -{ +function Locate-MSBuildPath { $vsInstallPath = Locate-VsInstallPath $msbuildPath = Get-ChildItem (Join-Path -path $vsInstallPath -childPath "MSBuild\*\Bin\MSBuild.exe") @@ -1054,66 +1121,61 @@ function Locate-MSBuildPath return $msBuild.FullName } -function Locate-VsInstallPath -{ - $vswhere = Join-Path -path $env:TP_PACKAGES_DIR -ChildPath "vswhere\$env:VSWHERE_VERSION\tools\vswhere.exe" - if (!(Test-Path -path $vswhere)) { - throw "Unable to locate vswhere in path '$vswhere'." - } - - Write-Verbose "Using '$vswhere' to locate VS installation path." - - $requiredPackageIds = @("Microsoft.Component.MSBuild", "Microsoft.Net.Component.4.6.TargetingPack", "Microsoft.VisualStudio.Component.VSSDK") - Write-Verbose "VSInstallation requirements : $requiredPackageIds" - - Try - { - if ($TPB_CIBuild) { - $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -products * -requires $requiredPackageIds -property installationPath" - } - else { - # Allow using pre release versions of VS for dev builds - $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -prerelease -products * -requires $requiredPackageIds -property installationPath" - } - } - Catch [System.Management.Automation.MethodInvocationException] - { - throw "Failed to find VS installation with requirements: $requiredPackageIds" - } - - if ($null -eq $vsInstallPath -or 0 -eq @($vsInstallPath).Count) { +function Locate-VsInstallPath { + $vswhere = Join-Path -path $env:TP_PACKAGES_DIR -ChildPath "vswhere\$env:VSWHERE_VERSION\tools\vswhere.exe" + if (!(Test-Path -path $vswhere)) { + throw "Unable to locate vswhere in path '$vswhere'." + } + + Write-Verbose "Using '$vswhere' to locate VS installation path." + + $requiredPackageIds = @("Microsoft.Component.MSBuild", "Microsoft.Net.Component.4.6.TargetingPack", "Microsoft.VisualStudio.Component.VSSDK") + Write-Verbose "VSInstallation requirements : $requiredPackageIds" + + Try { + if ($TPB_CIBuild) { + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -products * -requires $requiredPackageIds -property installationPath" + } + else { + # Allow using pre release versions of VS for dev builds + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -prerelease -products * -requires $requiredPackageIds -property installationPath" + } + } + Catch [System.Management.Automation.MethodInvocationException] { throw "Failed to find VS installation with requirements: $requiredPackageIds" - } - else { + } + + if ($null -eq $vsInstallPath -or 0 -eq @($vsInstallPath).Count) { + throw "Failed to find VS installation with requirements: $requiredPackageIds" + } + else { Write-Verbose "Found VS installation with requirements '$($requiredPackageIds -join "','")' : '$($vsInstallPath -join "','")'." - } + } - $vsPath = $vsInstallPath | Select-Object -First 1 - Write-Verbose "VSInstallPath is : $vsPath" - return $vsPath + $vsPath = $vsInstallPath | Select-Object -First 1 + Write-Verbose "VSInstallPath is : $vsPath" + return $vsPath } -function Update-VsixVersion($vsixProjectDir) -{ +function Update-VsixVersion($vsixProjectDir) { Write-Log "Update-VsixVersion: Started." $vsixVersion = $Version # Build number comes in the form 20170111-01(yyyymmdd-buildNoOfThatDay) # So Version of the vsix will be 15.1.0.2017011101 $vsixVersionSuffix = $BuildNumber.Split("-"); - if($vsixVersionSuffix.Length -ige 2) { + if ($vsixVersionSuffix.Length -ige 2) { $vsixVersion = "$vsixVersion.$($vsixVersionSuffix[0])$($vsixVersionSuffix[1])" } - $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | ForEach-Object {$_.ToString().Replace("`$version`$", "$vsixVersion") } + $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | ForEach-Object { $_.ToString().Replace("`$version`$", "$vsixVersion") } Set-Content -path "$vsixProjectDir\source.extension.vsixmanifest" -value $manifestContentWithVersion Write-Log "Update-VsixVersion: Completed." } -function Generate-Manifest ($PackageFolder) -{ - $packagesFolderName = [System.IO.Path]::GetFileName($PackageFolder) +function Generate-Manifest ($PackageFolder) { + $packagesFolderName = [System.IO.Path]::GetFileName($PackageFolder) Write-Log "Generate-Manifest ($packagesFolderName): Started." $generateManifestPath = Join-Path $env:TP_ROOT_DIR "scripts\build\GenerateManifest.proj" @@ -1124,8 +1186,7 @@ function Generate-Manifest ($PackageFolder) Write-Log "Generate-Manifest ($packagesFolderName): Completed." } -function Build-SpecificProjects -{ +function Build-SpecificProjects { Write-Log "Build-SpecificProjects: Started for pattern: $ProjectNamePatterns" # FrameworksAndOutDirs format ("", ""). $FrameworksAndOutDirs = ( @@ -1141,18 +1202,18 @@ function Build-SpecificProjects # Get projects to build. Get-ChildItem -Recurse -Path $env:TP_ROOT_DIR -Include *.csproj | ForEach-Object { foreach ($ProjectNamePattern in $ProjectNamePatterns) { - if($_.FullName -match $ProjectNamePattern) { - $ProjectsToBuild += ,"$_" + if ($_.FullName -match $ProjectNamePattern) { + $ProjectsToBuild += , "$_" } } } - if( $null -eq $ProjectsToBuild){ + if ( $null -eq $ProjectsToBuild) { Write-Error "No csproj name match for given pattern: $ProjectNamePatterns" } # Build Projects. - foreach($ProjectToBuild in $ProjectsToBuild) { + foreach ($ProjectToBuild in $ProjectsToBuild) { Write-Log "Building Project $ProjectToBuild" # Restore and Build $output = Invoke-Exe $dotnetPath -Arguments "restore $ProjectToBuild" @@ -1167,13 +1228,13 @@ function Build-SpecificProjects # Copy artifacts $ProjectDir = [System.IO.Path]::GetDirectoryName($ProjectToBuild) - foreach($FrameworkAndOutDir in $FrameworksAndOutDirs) { + foreach ($FrameworkAndOutDir in $FrameworksAndOutDirs) { $fromDir = $([System.IO.Path]::Combine($ProjectDir, "bin", $TPB_Configuration, $FrameworkAndOutDir[0])) $toDir = $([System.IO.Path]::Combine($env:TP_OUT_DIR, $TPB_Configuration, $FrameworkAndOutDir[1])) - if ( Test-Path $fromDir){ + if ( Test-Path $fromDir) { Write-Log "Copying artifacts from $fromDir to $toDir" Get-ChildItem $fromDir | ForEach-Object { - if(-not ($_.PSIsContainer)) { + if (-not ($_.PSIsContainer)) { Copy-Item $_.FullName $toDir } } @@ -1182,8 +1243,7 @@ function Build-SpecificProjects } } -if ($ProjectNamePatterns.Count -ne 0) -{ +if ($ProjectNamePatterns.Count -ne 0) { # Build Specific projects. Build-SpecificProjects Exit @@ -1238,7 +1298,8 @@ if ($Force -or $Steps -contains "PrepareAcceptanceTests") { if ($Script:ScriptFailed) { Write-Log "Build failed. {$(Get-ElapsedTime($timer))}" -Level "Error" Exit 1 -} else { +} +else { Write-Log "Build succeeded. {$(Get-ElapsedTime($timer))}" Exit 0 } diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 073a55e0bd..d57a941d3d 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -13,20 +13,47 @@ from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync --> 17.2.0-dev + 2.2.8 2.2.8 1.0.3-preview + 2.3.1 2.3.1 2.3.1 3.10.1 - 3.10.0 + 3.11.0 3.8.0 4.4.12 + + + [2.2.9-preview-20220210-07] + [2.2.8] + [2.2.7] + [2.1.0] + [2.1.0] + [1.4.0] + + + + + [17.2.0-preview-20220131-20] + [17.1.0] + [17.0.0] + [16.6.1] + [16.11.0] + [15.9.2] + + 5.11.0 5.0.0 diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index 34fd49a1e8..38b98bd805 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -47,7 +47,7 @@ Write-Verbose "Setup dotnet configuration." $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR -$env:NUGET_EXE_Version = "5.8.1" +$env:NUGET_EXE_Version = "6.0.0" $env:DOTNET_CLI_VERSION = $GlobalJson.tools.dotnet # $env:DOTNET_RUNTIME_VERSION = "LATEST" $env:VSWHERE_VERSION = "2.0.2" diff --git a/scripts/verify-nupkgs.ps1 b/scripts/verify-nupkgs.ps1 index a760af7300..7989cb7fbc 100644 --- a/scripts/verify-nupkgs.ps1 +++ b/scripts/verify-nupkgs.ps1 @@ -56,7 +56,9 @@ function Verify-Nuget-Packages($packageDirectory, $version) Write-Error "Number of files are not equal $unzipNugetPackageDir, expected: $($expectedNumOfFiles[$packageKey]) actual: $actualNumOfFiles" } - Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null + # Don't remove the directories after you unpacked them + # they are useful for reviewing what is in the package. + # Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null } Write-Log "Completed Verify-Nuget-Packages." diff --git a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs index 9bbbca6b84..fc251b1c00 100644 --- a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs +++ b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs @@ -357,7 +357,7 @@ public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken var ackPayload = _dataSerializer.DeserializePayload(ackMessage); if (!ackPayload.Attached) { - EqtTrace.Warning(ackPayload.ErrorMessage); + EqtTrace.Warning($"DesignModeClient.AttachDebuggerToProcess: Attaching to process failed: {ackPayload.ErrorMessage}"); } return ackPayload.Attached; diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs index a1abda01df..7386f07d06 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if NETFRAMEWORK using System.Collections.Generic; -#endif using System.Diagnostics; using System.IO; @@ -41,16 +39,11 @@ public ConsoleParameters(IFileHelper fileHelper) _fileHelper = fileHelper; } -#if NETFRAMEWORK - /// - /// TODO: Remove the #if when project is targeted to netstandard2.0 /// Environment variables to be set for the process /// public Dictionary EnvironmentVariables { get; set; } -#endif - /// /// Trace level for logs. /// diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt index cb8161d464..e9c4b6ee04 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt @@ -196,4 +196,6 @@ static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resour static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.InvalidFilePath.get -> string static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.ResourceManager.get -> System.Resources.ResourceManager static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.VsTestProcessExitedAbnormally.get -> string -virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void \ No newline at end of file +virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt index 2b1c581fe8..e69de29bb2 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt @@ -1,2 +0,0 @@ -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs index 48b804836e..323ff2d95a 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs @@ -121,7 +121,6 @@ public void StartProcess(ConsoleParameters consoleParameters) EqtTrace.Verbose("VsTestCommandLineWrapper: Process Start Info {0} {1}", info.FileName, info.Arguments); -#if NETFRAMEWORK if (consoleParameters.EnvironmentVariables != null) { info.EnvironmentVariables.Clear(); @@ -133,7 +132,7 @@ public void StartProcess(ConsoleParameters consoleParameters) } } } -#endif + try { _process = Process.Start(info); diff --git a/src/package/external/external.csproj b/src/package/external/external.csproj index ebead5a9dc..4a0e70027d 100644 --- a/src/package/external/external.csproj +++ b/src/package/external/external.csproj @@ -33,7 +33,7 @@ - + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 4829c3edcb..934b7de8e6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Linq; using Microsoft.TestPlatform.TestUtilities; @@ -51,6 +52,17 @@ public class AcceptanceTestBase : IntegrationTestBase public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET50 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET31 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1"; + public const string DEFAULT_RUNNER_NETFX = "net451"; + /// + /// Our current defaults for .NET and .NET Framework. + /// + public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; + public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; + public const string LATEST_TO_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATESTPREVIEW_TO_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATEST = "Latest"; + public const string LATESTSTABLE= "LatestStable"; + internal const string MSTEST = "MSTest"; public static string And(string left, string right) { @@ -61,6 +73,11 @@ public static string And(string left, string right) protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo) { + testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo; + // The order here matters, it changes how the resulting path is built when we resolve test dlls and other assets. + testEnvironment.DllInfos = new[] { runnerInfo.AdapterInfo, runnerInfo.TestHostInfo }.Where(d => d != null).ToList(); + testEnvironment.DebugInfo = runnerInfo.DebugInfo; + testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; @@ -96,17 +113,37 @@ protected string GetTargetFramworkForRunsettings() } /// - /// Default RunSettings + /// Empty runsettings, just with the RunSettings tag that we require. /// /// + public string GetEmptyRunsettings() + { + return ""; + } + + /// + /// Almost empty runsettings, just specifying the target framework from the currently set test environment. + /// public string GetDefaultRunSettings() { - string runSettingsXml = $@" - - - {FrameworkArgValue} - - "; + return GetRunSettingsWithTargetFramework(FrameworkArgValue); + } + + /// + /// Almost empty runsettings, just specifying the given target framework. + /// Use the overload without any parameters to get the target framework from the currently set test environment. + /// + /// + public string GetRunSettingsWithTargetFramework(string targetFramework) + { + string runSettingsXml = + $@" + + + {targetFramework} + + "; + return runSettingsXml; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 671a91618d..f740998847 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -64,7 +64,7 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -89,7 +89,7 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -114,7 +114,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump;CollectAlways=True"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -139,6 +139,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s"""); var env = new Dictionary @@ -162,6 +163,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump"""); var env = new Dictionary @@ -185,6 +187,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true"""); var env = new Dictionary @@ -206,6 +209,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); var env = new Dictionary @@ -227,6 +231,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("child-crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); InvokeVsTest(arguments); @@ -242,6 +247,7 @@ public void HangDumpChildProcesses(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index b7907cbdd6..a600a7bf0c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -28,7 +28,7 @@ public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( @@ -58,7 +58,7 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, @@ -87,7 +87,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetTestDllForFramework("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -113,8 +113,8 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); - var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll"); + var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs index fba250c5a2..a03280f37c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs @@ -20,13 +20,13 @@ public void RunningTestWithAFailingDebugAssertDoesNotCrashTheHostingProcess(Runn // is to not crash the process when we are running in debug, and debugger is attached SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); // this will have failed tests when our trace listener works and crash the testhost process when it does not // because crashing processes is what a failed Debug.Assert does by default, unless you have a debugger attached - ValidateSummaryStatus(passedTestsCount: 4, failedTestsCount: 4, 0); + ValidateSummaryStatus(passed: 4, failed: 4, 0); StringAssert.Contains(StdOut, "threw exception: Microsoft.VisualStudio.TestPlatform.TestHost.DebugAssertException:"); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 2f6087aa17..fa9bb8627b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -38,7 +38,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs index e60f46ed94..5ff96f6cd2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs @@ -78,7 +78,7 @@ public void GlobalInstallation() AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication(GetDefaultDotnetMuxerLocation, $"test {buildAssemblyPath} --arch x64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -148,7 +148,7 @@ public void DOTNET_ROOTS_EnvironmentVariables(bool dotnetRoot, bool dotnetRootX6 AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch x64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -192,7 +192,7 @@ public void PrivateX64BuildToGlobalArmInstallation() AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch arm64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -249,7 +249,7 @@ public void PrivateX64BuildToDOTNET_ROOTS_EnvironmentVariables(bool dotnetRoot, AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch arm64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs index ac0a468d21..05594bc697 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs @@ -36,7 +36,7 @@ public void RunDotnetTestWithDll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = GetAssetFullPath("SimpleTestProject.dll"); + var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll"); InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"""); // ensure our dev version is used @@ -68,7 +68,7 @@ public void RunDotnetTestWithDllPassInlineSettings(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = GetAssetFullPath("ParametrizedTestProject.dll"); + var assemblyPath = BuildMultipleAssemblyPath("ParametrizedTestProject.dll"); InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")"); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index b88bf2e02d..037ec5deda 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -6,6 +6,7 @@ using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; using TestPlatform.TestUtilities; @@ -16,21 +17,74 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] public class ExecutionTests : AcceptanceTestBase { + //TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] - [NetCoreTargetFrameworkDataSource] + [TestCategory("Windows-Review")] + [MSTestCompatibilityDataSource(InProcess = true)] public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + StdErrHasTestRunFailedMessageButNoOtherError(); + StdOutHasNoWarnings(); + } + + [TestMethod] + [TestCategory("Windows-Review")] + [TestPlatformCompatibilityDataSource()] + public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(passed: 2, failed: 2, skipped: 2); + ExitCodeEquals(1); // failing tests + StdErrHasTestRunFailedMessageButNoOtherError(); + StdOutHasNoWarnings(); + } + + [TestMethod] + [TestCategory("Windows-Review")] + [TestHostCompatibilityDataSource] + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); - InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests } + [TestMethod] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource] + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(RunnerInfo runnerInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + + // TODO: This one mixes different frameworks, I can make it work, but it is worth it? We are going to test + // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test + // frameworks have any added value over using 2 mstest dlls? [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] @@ -38,13 +92,13 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var xunitAssemblyPath = _testEnvironment.TargetFramework.Equals("net451") ? _testEnvironment.GetTestAsset("XUTestProject.dll", "net46") : _testEnvironment.GetTestAsset("XUTestProject.dll"); - assemblyPaths = string.Concat(assemblyPaths, "\" \"", xunitAssemblyPath); - InvokeVsTestForExecution(assemblyPaths, string.Empty, FrameworkArgValue, string.Empty); + assemblyPaths = string.Join(" ", assemblyPaths, xunitAssemblyPath.AddDoubleQuote()); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 1); ExitCodeEquals(1); // failing tests @@ -54,14 +108,13 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf // and after --arch feature implementation we won't find correct muxer on CI. [TestCategory("Windows")] [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] + [MSTestCompatibilityDataSource] public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); + var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); arguments += GetDiagArg(TempDirectory.Path); @@ -90,7 +143,7 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); @@ -110,7 +163,7 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -126,7 +179,7 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); @@ -153,7 +206,7 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -183,7 +236,7 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run File.Delete(diagLogFilePath); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -204,7 +257,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); @@ -273,7 +326,7 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -293,7 +346,7 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed @@ -312,7 +365,7 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -330,7 +383,7 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs index 763e0d5016..7548f73d3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs @@ -18,7 +18,7 @@ public void UITestShouldPassIfApartmentStateIsSTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod"); InvokeVsTest(arguments); @@ -32,7 +32,7 @@ public void WarningShouldBeShownWhenValueIsSTAForNetCore(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTest2 -- RunConfiguration.ExecutionThreadApartmentState=STA"); InvokeVsTest(arguments); @@ -47,7 +47,7 @@ public void UITestShouldFailWhenDefaultApartmentStateIsMTA(RunnerInfo runnerInfo SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=MTA"); InvokeVsTest(arguments); @@ -62,7 +62,7 @@ public void CancelTestExectionShouldWorkWhenApartmentStateIsSTA(RunnerInfo runne SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:UITestWithSleep1,UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=STA RunConfiguration.TestSessionTimeout=2000"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs new file mode 100644 index 0000000000..157b2a14b0 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -0,0 +1,413 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml; +using NuGet.Versioning; + +using Microsoft.TestPlatform.TestUtilities; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public class CompatibilityRowsBuilder +{ + private static XmlDocument? s_depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _runnerVersions; + private readonly string[] _hostFrameworks; + private readonly string[] _adapterVersions; + private readonly string[] _adapters; + private readonly string[] _hostVersions; + + public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) + { + _runnerFrameworks = runnerFrameworks.Split(';'); + _runnerVersions = runnerVersions.Split(';'); + _hostFrameworks = hostFrameworks.Split(';'); + _hostVersions = hostVersions.Split(';'); + _adapterVersions = adapterVersions.Split(';'); + _adapters = adapters.Split(';'); + } + + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool WithInProcess { get; set; } = true; + public bool WithEveryVersionOfRunner { get; set; } = true; + public bool WithEveryVersionOfHost { get; set; } = true; + public bool WithEveryVersionOfAdapter { get; set; } = true; + public bool WithOlderConfigurations { get; set; } = true; + + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + + public List CreateData() + { + var dataRows = new List(); + + if (WithEveryVersionOfRunner) + AddEveryVersionOfRunner(dataRows); + + if (WithEveryVersionOfHost) + AddEveryVersionOfHost(dataRows); + + if (WithEveryVersionOfAdapter) + AddEveryVersionOfAdapter(dataRows); + + if (WithOlderConfigurations) + AddOlderConfigurations(dataRows); + + if (WithInProcess) + AddInProcess(dataRows); + + var minVersion = SemanticVersion.Parse("0.0.0-alpha.1"); + var maxVersion = SemanticVersion.Parse("9999.0.0"); + SemanticVersion? beforeRunnerVersion = maxVersion; + SemanticVersion? afterRunnerVersion = minVersion; + SemanticVersion? beforeTestHostVersion = maxVersion; + SemanticVersion? afterTestHostVersion = minVersion; + SemanticVersion? beforeAdapterVersion = maxVersion; + SemanticVersion? afterAdapterVersion = minVersion; + + if (BeforeRunnerFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeRunnerFeature]; + beforeRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterRunnerFeature != null) + { + var feature = Features.TestPlatformFeatures[AfterRunnerFeature]; + afterRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + } + + if (BeforeTestHostFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeTestHostFeature]; + beforeTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterTestHostFeature != null) + { + var feature = Features.TestPlatformFeatures[AfterTestHostFeature]; + afterTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + } + + if (BeforeAdapterFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeAdapterFeature]; + beforeAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterAdapterFeature != null) + { + var feature = Features.AdapterFeatures[AfterAdapterFeature]; + afterAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); + } + + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Run .NET Framework tests only on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + // TODO: maybe we should throw if we don't end up generating any data + // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. + // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping + // compatibility. + + Func isInRange = (version, before, after) => version < before && after < version; + + var rows = dataRows.Where(r => r.VSTestConsoleInfo != null + && isInRange(SemanticVersion.Parse(r.VSTestConsoleInfo.Version), beforeRunnerVersion, afterRunnerVersion) + && r.TestHostInfo != null && isInRange(SemanticVersion.Parse(r.TestHostInfo.Version), beforeTestHostVersion, afterTestHostVersion) + && r.AdapterInfo != null && isInRange(SemanticVersion.Parse(r.AdapterInfo.Version), beforeAdapterVersion, afterAdapterVersion)).ToList(); + + // We use ToString to determine which values are unique. Not great solution, but works better than using records. + var distinctRows = new Dictionary(); + rows.ForEach(r => distinctRows[r.ToString()] = r); + + if (distinctRows.Count == 0) + { + // TODO: This needs to be way more specific about what happened. And possibly propagate as inconclusive state if we decide to update versions automatically? + throw new InvalidOperationException("There were no rows that matched the specified criteria."); + } + + return distinctRows.Values.ToList(); + } + + private void AddInProcess(List dataRows) + { + foreach (var runnerFramework in _runnerFrameworks) + { + if (!runnerFramework.StartsWith("net4")) + { + continue; + } + + foreach (var runnerVersion in _runnerVersions) + { + foreach (var adapter in _adapters) + { + foreach (var adapterVersion in _adapterVersions) + { + AddRow(dataRows, "In process", runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); + } + } + } + } + } + + private void AddOlderConfigurations(List dataRows) + { + // Older configurations where the runner, host and adapter version are the same. + // We already added the row where all are newest when adding combination with all runners. + foreach (var runnerVersion in _runnerVersions) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + var hostVersion = runnerVersion; + foreach (var adapter in _adapters) + { + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, "Older", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddEveryVersionOfAdapter(List dataRows) + { + var runnerVersion = _runnerVersions[0]; + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + foreach (var adapterVersion in _adapterVersions) + { + AddRow(dataRows, "Every adapter", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddEveryVersionOfHost(List dataRows) + { + var runnerVersion = _runnerVersions[0]; + + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always the same as the runner. There are no variations + // so we just need to add host versions for .NET testhosts. + var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.ToArray(); + foreach (var hostVersion in hostVersions) + { + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, "Every host", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddEveryVersionOfRunner(List dataRows) + { + foreach (var runnerVersion in _runnerVersions) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, "Every runner", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddRow(List dataRows, string batch, + string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) + { + RunnerInfo runnerInfo = GetRunnerInfo(batch, runnerFramework, hostFramework, inIsolation); + runnerInfo.DebugInfo = GetDebugInfo(); + runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); + + // The order in which we add them matters. We end up both modifying the same path + // and adding to it. So the first one added will be later in the path. E.g.: + // Adding testSdk first: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll + // versus adding testSdk second: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll + runnerInfo.TestHostInfo = GetNetTestSdkInfo(hostVersion); + runnerInfo.AdapterInfo = GetMSTestInfo(adapterVersion); + dataRows.Add(runnerInfo); + } + + private DebugInfo GetDebugInfo() + { + return new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTestHost = DebugTestHost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; + } + + private RunnerInfo GetRunnerInfo(string batch, string runnerFramework, string hostFramework, bool inIsolation) + { + return new RunnerInfo + { + Batch = batch, + RunnerFramework = runnerFramework, + TargetFramework = hostFramework, + InIsolationValue = inIsolation ? AcceptanceTestBase.InIsolation : null + }; + } + + private DllInfo GetMSTestInfo(string msTestVersion) + { + var depsXml = GetDependenciesXml(); + + // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new DllInfo + { + Name = "MSTest", + PropertyName = "MSTest", + VersionType = msTestVersion, + Version = version, + Path = versionSpecificBinPath, + }; + } + + private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST + ? "NETTestSdkVersion" + : $"VSTestConsole{vstestConsoleVersion}Version"; + + var packageName = runnerInfo.IsNetFrameworkRunner + ? "microsoft.testplatform" + : "microsoft.testplatform.cli"; + + // It is okay when node is null, we will fail to find the executable later, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; + var vstestConsolePath = runnerInfo.IsNetFrameworkRunner + ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") + : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); + + if (version.StartsWith("15.")) + { + vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); + } + + return new VSTestConsoleInfo + { + VersionType = vstestConsoleVersion, + Version = version, + Path = vstestConsolePath, + }; + } + + private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = testhostVersionType == AcceptanceTestBase.LATEST + ? "NETTestSdkVersion" + : $"VSTestConsole{testhostVersionType}Version"; + + // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // + // We use the VSTestConsole properties to figure out testhost version, for now. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; + + return new NetTestSdkInfo + { + VersionType = testhostVersionType, + Version = version, + Path = versionSpecificBinPath + }; + } + + private static XmlDocument GetDependenciesXml() + { + if (s_depsXml != null) + return s_depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + s_depsXml = depsXml; + return depsXml; + } +} + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs new file mode 100644 index 0000000000..9014349f06 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public class Feature +{ + public Feature(string version, string issue) + { + Version = version; + Issue = issue; + } + + public string Version { get; } + public string Issue { get; } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs new file mode 100644 index 0000000000..0be9ac104c --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Collections.Immutable; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public static class Features +{ + public const string ATTACH_DEBUGGER_FLOW = nameof(ATTACH_DEBUGGER_FLOW); + public const string MSTEST_EXAMPLE_FEATURE = nameof(MSTEST_EXAMPLE_FEATURE); + + public static IImmutableDictionary TestPlatformFeatures { get; } = new Dictionary + { + [ATTACH_DEBUGGER_FLOW] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), + }.ToImmutableDictionary(); + + public static IImmutableDictionary AdapterFeatures { get; internal set; } = new Dictionary + { + [MSTEST_EXAMPLE_FEATURE] = new("2.2.8", issue: "This feature does not actually exist."), + }.ToImmutableDictionary(); +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/IsExternalInit.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/IsExternalInit.cs similarity index 100% rename from test/Microsoft.TestPlatform.AcceptanceTests/IsExternalInit.cs rename to test/Microsoft.TestPlatform.AcceptanceTests/Extension/IsExternalInit.cs diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs new file mode 100644 index 0000000000..f6d20b577c --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public class MSTestCompatibilityDataSource : TestDataSource +{ + private readonly CompatibilityRowsBuilder _builder; + + public MSTestCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) + { + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + // runner versions + AcceptanceTestBase.LATEST_TO_LEGACY, + hostFrameworks, + // host versions + AcceptanceTestBase.LATEST_TO_LEGACY, + adapterVersions, + // adapters + AcceptanceTestBase.MSTEST); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool InProcess { get; set; } + + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } + + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public override void CreateData(MethodInfo methodInfo) + { + _builder.WithEveryVersionOfRunner = false; + _builder.WithEveryVersionOfHost = false; + _builder.WithEveryVersionOfAdapter = true; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = InProcess; + + _builder.BeforeRunnerFeature = BeforeRunnerFeature; + _builder.AfterRunnerFeature = AfterRunnerFeature; + + _builder.BeforeTestHostFeature = BeforeTestHostFeature; + _builder.AfterTestHostFeature = AfterTestHostFeature; + + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTestHost = DebugTestHost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 67cc4fdc68..202264d0c7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -23,27 +23,47 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetCoreRunner : Attribute, ITestDataSource { + private readonly string _targetFrameworks; + /// /// Initializes a new instance of the class. /// /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { + _targetFrameworks = targetFrameworks; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); // on non-windows we want to filter down only to netcoreapp runner, and net5.0 and newer. Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - foreach (var fmw in targetFrameworks.Split(';').Where(filter)) + foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw) }); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.CoreRunnerFramework, + TargetFramework = fmw, + InIsolationValue = null, + }; + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTestHost = DebugTestHost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } - } - - private readonly List _dataRows = new(); - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 4e8ae4ae9d..56ff01036b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -22,7 +22,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetCoreTargetFrameworkDataSource : Attribute, ITestDataSource { - private readonly List _dataRows = new(); + private readonly bool _useDesktopRunner; + private readonly bool _useCoreRunner; + private readonly bool _useNetCore21Target; + private readonly bool _useNetCore31Target; + /// /// Initializes a new instance of the class. /// @@ -38,45 +42,68 @@ public NetCoreTargetFrameworkDataSource( // all tests to avoid changing all acceptance tests right now bool useNetCore31Target = false) { + _useDesktopRunner = useDesktopRunner; + _useCoreRunner = useCoreRunner; + _useNetCore21Target = useNetCore21Target; + _useNetCore31Target = useNetCore31Target; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) + { + var runnerInfo = new RunnerInfo + { + RunnerFramework = runnerFramework, + TargetFramework = targetFramework, + InIsolationValue = null + }; + runnerInfo.DebugInfo = new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTestHost = DebugTestHost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); + } + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - if (useDesktopRunner && isWindows) + if (_useDesktopRunner && isWindows) { var runnerFramework = IntegrationTestBase.DesktopRunnerFramework; - if (useNetCore21Target) + if (_useNetCore21Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core21TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core21TargetFramework); } - if (useNetCore31Target) + if (_useNetCore31Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core31TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core31TargetFramework); } } - if (useCoreRunner) + if (_useCoreRunner) { var runnerFramework = IntegrationTestBase.CoreRunnerFramework; - if (useNetCore21Target) + if (_useNetCore21Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core21TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core21TargetFramework); } - if (useNetCore31Target) + if (_useNetCore31Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core31TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core31TargetFramework); } } - } - - private void AddRunnerDataRow(string runnerFramework, string targetFramework) - { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework); - _dataRows.Add(new object[] { runnerInfo }); - } - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index 49f33f8d02..0f1faa65f5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -28,24 +28,45 @@ public class NetFrameworkRunner : Attribute, ITestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetFrameworkRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { + _targetFrameworks = targetFrameworks; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + private readonly string _targetFrameworks; + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (!isWindows) { - return; + return dataRows; } - foreach (var fmw in targetFrameworks.Split(';')) + foreach (var fmw in _targetFrameworks.Split(';')) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation) }); - } - - } + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = fmw, + InIsolationValue = AcceptanceTestBase.InIsolation + }; + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTestHost = DebugTestHost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; - private readonly List _dataRows = new(); + dataRows.Add(new object[] { runnerInfo }); + } - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index 626d11422f..b809da7799 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -22,6 +22,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetFullTargetFrameworkDataSource : Attribute, ITestDataSource { + private readonly bool _inIsolation; + private readonly bool _inProcess; + private readonly bool _useDesktopRunner; + private readonly bool _useCoreRunner; + /// /// Initializes a new instance of the class. /// @@ -31,60 +36,79 @@ public class NetFullTargetFrameworkDataSource : Attribute, ITestDataSource /// To run tests with core runner(dotnet vstest.console.dll) public NetFullTargetFrameworkDataSource(bool inIsolation = true, bool inProcess = false, bool useDesktopRunner = true, bool useCoreRunner = true) { - _dataRows = new List(); + _inIsolation = inIsolation; + _inProcess = inProcess; + _useDesktopRunner = useDesktopRunner; + _useCoreRunner = useCoreRunner; + } - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - if (useCoreRunner && isWindows) - { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework) }); - } + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; - if (useDesktopRunner && isWindows) + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + if (_useCoreRunner && isWindows) { - if (inIsolation) + var runnerInfo = new RunnerInfo { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation) }); - } - - if (inProcess) + RunnerFramework = IntegrationTestBase.CoreRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = null + }; + runnerInfo.DebugInfo = new DebugInfo { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework) }); - } + DebugVSTestConsole = DebugVSTestConsole, + DebugTestHost = DebugTestHost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } - } - - /// - /// Initializes a new instance of the class. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public NetFullTargetFrameworkDataSource(string[] targetFrameworks, bool inIsolation = true, bool inProcess = false) - { - if (inIsolation) + if (_useDesktopRunner && isWindows) { - foreach (var fmw in targetFrameworks) + if (_inIsolation) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation) }); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = AcceptanceTestBase.InIsolation + }; + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTestHost = DebugTestHost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } - } - if (inProcess) - { - foreach (var fmw in targetFrameworks) + if (_inProcess) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw) }); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = null + }; + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTestHost = DebugTestHost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } } - } - - /// - /// Gets or sets the data rows. - /// - private readonly List _dataRows = new(); - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs new file mode 100644 index 0000000000..357dcd1070 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +/// +/// A data source that provides every version of runner. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class RunnerCompatibilityDataSource : TestDataSource +{ + private readonly CompatibilityRowsBuilder _builder; + + public RunnerCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET) + { + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + runnerVersions, + hostFrameworks, + // host versions + AcceptanceTestBase.LATEST, + // adapter versions + AcceptanceTestBase.LATESTSTABLE, + // adapters + AcceptanceTestBase.MSTEST); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool InProcess { get; set; } + + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + + //public string? BeforeTestHostFeature { get; set; } + //public string? AfterTestHostFeature { get; set; } + + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public override void CreateData(MethodInfo methodInfo) + { + _builder.WithEveryVersionOfRunner = true; + _builder.WithEveryVersionOfHost = false; + _builder.WithEveryVersionOfAdapter = false; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = InProcess; + + _builder.BeforeRunnerFeature = BeforeFeature; + _builder.AfterRunnerFeature = AfterFeature; + + //_builder.BeforeTestHostFeature = BeforeTestHostFeature; + //_builder.AfterTestHostFeature = AfterTestHostFeature; + + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTestHost = DebugTestHost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); + } +} + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index 75462340d6..6e1fefc769 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -2,6 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Linq; + +using Microsoft.TestPlatform.TestUtilities; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -12,8 +15,18 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// Supported value = /InIsolation. [Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer -public record RunnerInfo(string RunnerFramework, string TargetFramework, string InIsolationValue = "") +public class RunnerInfo { + public string? RunnerFramework { get; set; } + public VSTestConsoleInfo? VSTestConsoleInfo { get; set; } + public string? TargetFramework { get; set; } + public string? InIsolationValue { get; set; } + public DebugInfo? DebugInfo { get; set; } + public NetTestSdkInfo? TestHostInfo { get; set; } + public DllInfo? AdapterInfo { get; set; } + + public string? Batch { get; set; } + /// /// Is running via .NET "Core" vstest.console? /// @@ -22,7 +35,7 @@ public record RunnerInfo(string RunnerFramework, string TargetFramework, string /// /// Is running via .NET Framework vstest.console? /// - public bool IsNetFrameworkRunner => RunnerFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); + public bool IsNetFrameworkRunner => RunnerFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); /// /// Is running via .NET "Core" testhost? @@ -32,15 +45,19 @@ public record RunnerInfo(string RunnerFramework, string TargetFramework, string /// /// Is running via .NET Framework testhost? /// - public bool IsNetFrameworkTarget => TargetFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); + public bool IsNetFrameworkTarget => TargetFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); public override string ToString() - => string.Join( - ",", - new[] - { - "RunnerFramework = " + RunnerFramework, - " TargetFramework = " + TargetFramework, - string.IsNullOrEmpty(InIsolationValue) ? " InProcess" : " InIsolation", - }); + { + return string.Join(", ", new[] + { + Batch != null ? $"{Batch}" : null, + $"Runner = {RunnerFramework}", + $"TargetFramework = {TargetFramework}", + string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation", + VSTestConsoleInfo == null ? null : VSTestConsoleInfo.ToString(), + TestHostInfo == null ? null : string.Join(",", TestHostInfo), + AdapterInfo == null ? null : string.Join(",", AdapterInfo) + }.Where(s => s != null)); + } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs new file mode 100644 index 0000000000..d041d36e79 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Reflection; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource where T1 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1) + { + _data.Add(new object[] { value1 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1) + { + return $"{methodInfo.Name} ({value1})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0]); + } +} + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2) + { + _data.Add(new object[] { value1, value2 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2) + { + return $"{methodInfo.Name} ({value1}, {value2})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1]); + } +} + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull + where T3 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2, T3 value3) + { + _data.Add(new object[] { value1, value2, value3 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2, T3 value3) + { + return $"{methodInfo.Name} ({value1}, {value2}, {value3})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2]); + } +} + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull + where T3 : notnull + where T4 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2, T3 value3, T4 value4) + { + _data.Add(new object[] { value1, value2, value3, value4 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2, T3 value3, T4 value4) + { + return $"{methodInfo.Name} ({value1}, {value2}, {value3}, {value4})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2], (T4)data[3]); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs new file mode 100644 index 0000000000..5f117fff0b --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +/// +/// A data source that provides a huge mix of runners, hosts and mstest adapter, to add up >100 tests. +/// You can control which runner versions, host versions, and adapter versions will be used. This should be +/// used only to test the most common scenarios, or special configurations that are candidates for their own +/// specialized source. +/// +/// By default net451 and netcoreapp2.1 are used for both runner and host. (4 combinations) +/// Then run with every version of runner is added. +/// Then run with every version of test.sdk is added. +/// Then run with every combination of testhost and adapter is added. +/// And then run in process is added. +/// +/// All of those are filtered down to have no duplicates, and to pass the +/// Before and After platform version filters, and adapter filters. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class TestPlatformCompatibilityDataSource : TestDataSource +{ + private readonly CompatibilityRowsBuilder _builder; + + public TestPlatformCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) + { + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder(runnerFrameworks, runnerVersions, hostFrameworks, hostVersions, adapterVersions, adapters); + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool WithInProcess { get; set; } = true; + + public bool WithEveryVersionOfRunner { get; set; } = true; + + public bool WithEveryVersionOfHost { get; set; } = true; + + public bool WithEveryVersionOfAdapter { get; set; } = true; + + public bool WithOlderConfigurations { get; set; } = true; + + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } + + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public override void CreateData(MethodInfo methodInfo) + { + _builder.WithEveryVersionOfRunner = WithEveryVersionOfRunner; + _builder.WithEveryVersionOfHost = WithEveryVersionOfHost; + _builder.WithEveryVersionOfAdapter = WithEveryVersionOfAdapter; + _builder.WithOlderConfigurations = WithOlderConfigurations; + _builder.WithInProcess = WithInProcess; + + _builder.BeforeRunnerFeature = BeforeRunnerFeature; + _builder.AfterRunnerFeature = AfterRunnerFeature; + + _builder.BeforeTestHostFeature = BeforeTestHostFeature; + _builder.AfterTestHostFeature = AfterTestHostFeature; + + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTestHost = DebugTestHost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs new file mode 100644 index 0000000000..4bcbba2f93 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +/// +/// A data source that provides every testhost. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class TestHostCompatibilityDataSource : TestDataSource +{ + private readonly CompatibilityRowsBuilder _builder; + + public TestHostCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY) + { + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + // runner versions + AcceptanceTestBase.LATEST, + hostFrameworks, + hostVersions, + // adapter versions + AcceptanceTestBase.LATESTSTABLE, + // adapter + AcceptanceTestBase.MSTEST); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + + public override void CreateData(MethodInfo methodInfo) + { + _builder.WithEveryVersionOfRunner = false; + _builder.WithEveryVersionOfHost = true; + _builder.WithEveryVersionOfAdapter = false; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = false; + + _builder.BeforeTestHostFeature = BeforeFeature; + _builder.AfterTestHostFeature = AfterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTestHost = DebugTestHost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 4ab4715e72..3d8f36218b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -92,7 +92,7 @@ public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerI { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); testAssembly = testAssembly.Replace("SimpleTestProject2.dll", "*TestProj*.dll"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 543b5b5acc..ef1e78ee2c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -28,7 +28,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -36,6 +36,7 @@ + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs index 7b1dff7eb9..70a132de3a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs @@ -23,7 +23,7 @@ public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Tell the test project which target framework we are expecting it to run as. @@ -35,6 +35,6 @@ public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runn InvokeVsTest(arguments, env); - ValidateSummaryStatus(passedTestsCount: 1, failedTestsCount: 0, 0); + ValidateSummaryStatus(passed: 1, failed: 0, 0); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index ff1287fa2e..d5f227fccf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -6,7 +6,6 @@ using System.Linq; using Microsoft.TestPlatform.TestUtilities; -using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; using Microsoft.VisualStudio.TestTools.UnitTesting; #nullable disable @@ -44,7 +43,7 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty); @@ -65,37 +64,4 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) ValidateDiscoveredTests(listOfTests); ExitCodeEquals(0); } - - public override string GetConsoleRunnerPath() - { - string consoleRunnerPath = string.Empty; - - if (IsDesktopRunner()) - { - consoleRunnerPath = Path.Combine(s_portablePackageFolder, "tools", "net451", "vstest.console.exe"); - } - else if (IsNetCoreRunner()) - { - var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; - consoleRunnerPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); - } - else - { - Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); - } - - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); - return consoleRunnerPath; - } - - protected override string SetVSTestConsoleDLLPathInArgs(string args) - { - var vstestConsoleDll = Path.Combine(s_portablePackageFolder, "tools", "netcoreapp2.1", "vstest.console.dll"); - vstestConsoleDll = vstestConsoleDll.AddDoubleQuote(); - args = string.Concat( - vstestConsoleDll, - " ", - args); - return args; - } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs index e82ddc9520..13cd916a56 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs @@ -22,7 +22,7 @@ public void WhenTestHostProcessExitsBecauseTheTargetedRuntimeIsNoFoundThenTheMes // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); const string testAssetProjectName = "SimpleTestProjectMessedUpTargetFramework"; - var assemblyPath = GetAssetFullPath(testAssetProjectName + ".dll", Core21TargetFramework); + var assemblyPath = GetTestDllForFramework(testAssetProjectName + ".dll", Core21TargetFramework); UpdateRuntimeConfigJsonWithInvalidFramework(assemblyPath, testAssetProjectName); // Act diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 0bda43d461..8ac882bb66 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -537,7 +537,7 @@ private void RunTestWithRunSettings(Dictionary runConfigurationD string runSettingsArgs, string additionalArgs, IEnumerable testhostProcessNames, int expectedNumOfProcessCreated) { - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var runsettingsPath = string.Empty; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs index c7dc6b7804..ed5e10fc8b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs @@ -27,10 +27,10 @@ public void RunningApplicationThatIsBuiltAsSelfContainedWillNotFailToFindHostpol SetTestEnvironment(_testEnvironment, runnerInfo); // the app is published to win10-x64 because of the runtime identifier in the project - var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); - ValidateSummaryStatus(passedTestsCount: 1, 0, 0); + ValidateSummaryStatus(passed: 1, 0, 0); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index 6125803ad0..d3304c795d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -64,7 +64,7 @@ public void RunMultipleTestAssembliesWithCodeCoverage(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = CreateCodeCoverageArguments(runnerInfo, assemblyPaths, out var trxFilePath); InvokeVsTest(arguments); @@ -85,7 +85,7 @@ public override string GetConsoleRunnerPath() consoleRunnerPath = Path.Combine(s_nugetPackageFolder, "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe"); } - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: \"{0}\"", consoleRunnerPath); return consoleRunnerPath; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index efa4b69597..0b56578e3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -11,7 +11,6 @@ using Castle.Core.Internal; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -28,13 +27,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private TempDirectory _tempDirectory; private RunEventHandler _runEventHandler; private TestRunAttachmentsProcessingEventHandler _testRunAttachmentsProcessingEventHandler; private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _tempDirectory); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); _testRunAttachmentsProcessingEventHandler = new TestRunAttachmentsProcessingEventHandler(); } @@ -185,6 +183,7 @@ await _vstestConsoleWrapper.ProcessTestRunAttachmentsAsync( [NetCoreTargetFrameworkDataSource] public async Task TestRunWithCodeCoverageAndAttachmentsProcessingNoMetrics(RunnerInfo runnerInfo) { + // System.Environment.SetEnvironmentVariable("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); // arrange SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); @@ -493,7 +492,7 @@ private void AssertCoverageResults(IList attachments) { foreach (var attachment in attachmentSet.Attachments) { - var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, _tempDirectory); + var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, TempDirectory); foreach (var project in GetProjects()) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs deleted file mode 100644 index e7fe325916..0000000000 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Diagnostics; -using System.Threading; - -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; - -#nullable disable - -namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; - -/// -/// The custom test host launcher. -/// -public class CustomTestHostLauncher : ITestHostLauncher2 -{ - public int ProcessId - { - get; - private set; - } - - /// - public bool IsDebug => true; - - public bool AttachDebuggerToProcess(int pid) => AttachDebuggerToProcess(pid, CancellationToken.None); - - public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) => true; - - /// - public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) - { - return LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); - } - - /// - public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) - { - var processInfo = new ProcessStartInfo( - defaultTestHostStartInfo.FileName, - defaultTestHostStartInfo.Arguments) - { - WorkingDirectory = defaultTestHostStartInfo.WorkingDirectory - }; - processInfo.UseShellExecute = false; - - var process = new Process { StartInfo = processInfo }; - process.Start(); - - return process != null ? process.Id : -1; - } -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 829b2e7eda..ac6b33001a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -1,11 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using System.Threading; + +using FluentAssertions; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; #nullable disable @@ -19,13 +23,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CustomTestHostTests : AcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private RunEventHandler _runEventHandler; - - private void Setup() - { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); - _runEventHandler = new RunEventHandler(); - } [TestCleanup] public void Cleanup() @@ -33,34 +30,172 @@ public void Cleanup() _vstestConsoleWrapper?.EndSession(); } + [TestMethod] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + // This does not work with testhosts that are earlier than when the feature was introduced, + // when latest runner is used, because the latest runner does not downgrade the messages when + // older testhost launcher is used. + // [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) + { + // Pins the existing functionality. + + // Arrange + SetTestEnvironment(_testEnvironment, runnerInfo); + + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + + // Act + var customTestHostLauncher = new TestHostLauncherV1(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); + + // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + + // Ensure we tried to launch testhost process. + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + } + + [TestMethod] + [TestCategory("Windows-Review")] + // [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource("net451", "netcoreapp2.1", "LegacyStable", BeforeFeature = Features.ATTACH_DEBUGGER_FLOW, DebugVSTestConsole = true)] + [Ignore("This is not working for any testhost prior 16.7.0 where the change was introduced. The launch testhost flow was replaced with AttachDebugger in runner, and the new callback to AttachDebugger happens in testhost." + + "But any testhost prior 16.7.0 where the change was introduced does not know to call back AttachDebugger, and the call never happens.")] + // You can confirm that the functionality broke between runner and testhost, past this point by using newer runners, against older testhosts. + // [TestPlatformCompatibilityDataSource(AfterRunnerFeature = Features.ATTACH_DEBUGGER_FLOW, BeforeTestHostFeature = Features.ATTACH_DEBUGGER_FLOW)] + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo) + { + // Ensures compatibility with testhost and runners that were created before 16.3.0. It makes sure that even if user provides + // an implementation of the ITestHostLauncher2 interface, then testhost expecting ITestHostLauncher still works correctly. + + // Arrange + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + + // Act + var customTestHostLauncher = new TestHostLauncherV2(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); + + // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + customTestHostLauncher.AttachDebuggerProcessId.Should().BeNull("we should not be asked to attach to a debugger, that flow is not used when vstest.console does not support it yet, even when it is given ITestHostLauncher2"); + } + + [TestMethod] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + // [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) + { + + // Arrange + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + + // Act + var customTestHostLauncher = new TestHostLauncherV2(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); + + // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.AttachDebuggerProcessId.Should().NotBeNull("we should be asked to attach a debugger to some process and save the pid of the process"); + customTestHostLauncher.LaunchProcessProcessId.Should().BeNull("we should not be asked to launch some real process, that flow is not used when vstest.console supports it and is given ITestHostLauncher2"); + } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunTestsWithCustomTestHostLaunch(RunnerInfo runnerInfo) + [TestCategory("Windows-Review")] + [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + + "in translation layer, and that just silently skips the call.")] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo) { + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); - var customTestHostLauncher = new CustomTestHostLauncher(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + // Act + var customTestHostLauncher = new TestHostLauncherV1(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert - Assert.AreEqual(6, _runEventHandler.TestResults.Count); - Assert.IsTrue(customTestHostLauncher.ProcessId != -1); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + + customTestHostLauncher.Should().NotBeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + } + + private static void EnsureTestsRunWithoutErrors(RunEventHandler runEventHandler, int passed, int failed, int skipped) + { + runEventHandler.Errors.Should().BeEmpty(); + runEventHandler.TestResults.Should().HaveCount(passed + failed + skipped); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(passed); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(failed); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(skipped); } - private IList GetTestAssemblies() + /// + /// The custom test host launcher implementing ITestHostLauncher. + /// + private class TestHostLauncherV1 : ITestHostLauncher { - var testAssemblies = new List + public int? LaunchProcessProcessId { get; private set; } + + /// + public bool IsDebug => true; + + /// + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) + { + return LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); + } + + /// + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") - }; + var processInfo = new ProcessStartInfo( + defaultTestHostStartInfo.FileName, + defaultTestHostStartInfo.Arguments) + { + WorkingDirectory = defaultTestHostStartInfo.WorkingDirectory + }; + processInfo.UseShellExecute = false; + + var process = new Process { StartInfo = processInfo }; + process.Start(); + + LaunchProcessProcessId = process?.Id; + return LaunchProcessProcessId ?? -1; + } + } + + /// + /// The custom test host launcher implementing ITestHostLauncher2, and through that also ITestHostLauncher. + /// + private class TestHostLauncherV2 : TestHostLauncherV1, ITestHostLauncher2 + { - return testAssemblies; + public int? AttachDebuggerProcessId { get; private set; } + + public bool AttachDebuggerToProcess(int pid) => AttachDebuggerToProcess(pid, CancellationToken.None); + + public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) + { + AttachDebuggerProcessId = pid; + return true; + } } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index 1c217dc762..cfae534f62 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -26,7 +26,7 @@ public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index b982311927..8fb4ce2b9a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -32,7 +32,7 @@ public class DiscoverTests : AcceptanceTestBase public void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _discoveryEventHandler = new DiscoveryEventHandler(); _discoveryEventHandler2 = new DiscoveryEventHandler2(); } @@ -44,30 +44,37 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); + _discoveryEventHandler = new DiscoveryEventHandler(); + _discoveryEventHandler2 = new DiscoveryEventHandler2(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + vstestConsoleWrapper.DiscoverTests(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource()] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - _vstestConsoleWrapper.DiscoverTests( - GetTestAssemblies(), + _discoveryEventHandler = new DiscoveryEventHandler(); + _discoveryEventHandler2 = new DiscoveryEventHandler2(); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + vstestConsoleWrapper.DiscoverTests( + GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, _discoveryEventHandler2); @@ -256,7 +263,7 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, runSettingsXml, discoveryEvents.Object)); // Assert. - Assert.IsTrue(isTestCancelled); + Assert.IsTrue(isTestCancelled, "Discovery was not cancelled"); var done = sw.Elapsed; var timeTillCancelled = done - cancellationCalled; timeTillCancelled.Should().BeLessThan(2.Seconds()); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs index b02ecf0730..848dff8996 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs @@ -117,7 +117,7 @@ public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessSta return -1; } - public bool AttachDebuggerToProcess(int pid) + public virtual bool AttachDebuggerToProcess(int pid) { // No op return true; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs index 5faeaa83d2..0beac2e85c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -21,7 +21,7 @@ public class LiveUnitTestingTests : AcceptanceTestBase public void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _discoveryEventHandler = new DiscoveryEventHandler(); _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs index 2aeb7a7680..73b07981b6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs @@ -23,7 +23,7 @@ public class RunSelectedTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); _discoveryEventHandler = new DiscoveryEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index f8b15f77c9..b17f17b57a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -29,7 +29,7 @@ public class RunTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } @@ -40,20 +40,21 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); - - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + vstestConsoleWrapper.RunTests(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler); // Assert - Assert.AreEqual(6, _runEventHandler.TestResults.Count); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + Assert.AreEqual(6, runEventHandler.TestResults.Count); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); } [TestMethod] @@ -188,12 +189,10 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo private IList GetTestAssemblies() { - var testAssemblies = new List + return new List { GetAssetFullPath("SimpleTestProject.dll"), GetAssetFullPath("SimpleTestProject2.dll") }; - - return testAssemblies; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 0da7554f11..4f6e641fb2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -31,8 +31,8 @@ public class RunTestsWithDifferentConfigurationTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out var logsDir); - _logsDir = logsDir; + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); + _logsDir = TempDirectory; _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 694f23409f..162ed10bde 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -24,7 +24,7 @@ public class RunTestsWithFilterTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } @@ -35,22 +35,22 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] + [TestCategory("Windows-Review")] + [RunnerCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - var sources = new List - { - GetAssetFullPath("SimpleTestProject.dll") - }; + _runEventHandler = new RunEventHandler(); - _vstestConsoleWrapper.RunTests( + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var sources = new List { GetAssetFullPath("MSTestProject1.dll") }; + + vstestConsoleWrapper.RunTests( sources, GetDefaultRunSettings(), - new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, + new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=MSTestProject1.UnitTest1.PassingTest" }, _runEventHandler); // Assert @@ -66,10 +66,7 @@ public void RunTestsWithFastFilter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List - { - GetAssetFullPath("SimpleTestProject.dll") - }; + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, diff --git a/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj b/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj index 782abe157e..cf33f901f3 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj +++ b/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj @@ -13,7 +13,7 @@ - + diff --git a/test/Microsoft.TestPlatform.PerformanceTests/TranslationLayer/ExecutionPerfTests.cs b/test/Microsoft.TestPlatform.PerformanceTests/TranslationLayer/ExecutionPerfTests.cs index c5fbdd54c2..c3bf68c9a8 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/TranslationLayer/ExecutionPerfTests.cs +++ b/test/Microsoft.TestPlatform.PerformanceTests/TranslationLayer/ExecutionPerfTests.cs @@ -35,7 +35,7 @@ public void RunTests(string projectName, double expectedNumberOfTests) var perfAnalyzer = new PerfAnalyzer(); using (perfAnalyzer.Start()) { - var vstestConsoleWrapper = GetVsTestConsoleWrapper(logFileDir: null, traceLevel: System.Diagnostics.TraceLevel.Off); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(traceLevel: System.Diagnostics.TraceLevel.Off); vstestConsoleWrapper.RunTests(GetPerfAssetFullPath(projectName), GetDefaultRunSettings(), options, runEventHandler); vstestConsoleWrapper.EndSession(); @@ -71,7 +71,7 @@ public void RunTestsWithDefaultAdaptersSkipped(string projectName, double expect var perfAnalyzer = new PerfAnalyzer(); using (perfAnalyzer.Start()) { - var vstestConsoleWrapper = GetVsTestConsoleWrapper(logFileDir: null, traceLevel: System.Diagnostics.TraceLevel.Off); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(traceLevel: System.Diagnostics.TraceLevel.Off); vstestConsoleWrapper.RunTests(GetPerfAssetFullPath(projectName), GetDefaultRunSettings(), options, runEventHandler); vstestConsoleWrapper.EndSession(); } diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs index 7fd3915a62..b18464992d 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs @@ -142,7 +142,7 @@ public void GetTestHostProcessStartInfoShouldIncludeTestSourcePathInArgumentsIfN { _testHostManager.Initialize(_mockMessageLogger.Object, $" {Architecture.X86} {Framework.DefaultFramework} {true} "); var connectionInfo = new TestRunnerConnectionInfo { Port = 123, ConnectionInfo = new TestHostConnectionInfo { Endpoint = "127.0.0.0:123", Role = ConnectionRole.Client, Transport = Transport.Sockets }, RunnerProcessId = 101 }; - var source = "C:\temp\a.dll"; + var source = @"C:\temp\a.dll"; var info = _testHostManager.GetTestHostProcessStartInfo( new List() { source }, @@ -158,7 +158,7 @@ public void GetTestHostProcessStartInfoShouldUseMonoAsHostOnNonWindowsIfNotStart _mockProcessHelper.Setup(p => p.GetCurrentProcessFileName()).Returns("/usr/bin/dotnet"); _mockEnvironment.Setup(e => e.OperatingSystem).Returns(PlatformOperatingSystem.Unix); _mockDotnetHostHelper.Setup(d => d.GetMonoPath()).Returns("/usr/bin/mono"); - var source = "C:\temp\a.dll"; + var source = @"C:\temp\a.dll"; var info = _testHostManager.GetTestHostProcessStartInfo( new List() { source }, diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs new file mode 100644 index 0000000000..f9ac35d770 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +[Serializable] +// For data source to serialize correctly to enable splitting testcases to one per test in VS, +// this must be serializable. This is sealed because the exact type must be used, not any child type. +// Otherwise it works, but silently does not split the test cases anymore. +public sealed class DebugInfo +{ + public bool DebugVSTestConsole { get; set; } + public bool DebugTestHost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs new file mode 100644 index 0000000000..b5d0590b65 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +// For data source to serialize correctly to enable splitting testcases to one per test in VS, +// this must be serializable. This is NOT sealed because we need this for adapters and testSdk. +// But be aware that the exact type must be used, not any child type for the data on data source object (RunnerInfo). +// Otherwise it works, but silently does not split the test cases anymore. +[Serializable] +public class DllInfo +{ + public string? Name { get; set; } + public string? PropertyName { get; set; } + public string? VersionType { get; set; } + public string? Version { get; set; } + public string? Path { get; set; } + + public override string ToString() => $" {Name} = {Version} [{VersionType}]"; + + public string UpdatePath(string path) + { + // Version is not directly used, below, but if it is not populated the path will be incorrect. + // We don't want to throw when creating SourcePathInfo because that is happening too early, and has worse error reporting. + if (Version == null) + throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for {PropertyName}{VersionType}Version."); + + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index e13e6e3194..c0e317d0f6 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -10,6 +11,8 @@ using System.Text.RegularExpressions; using System.Xml; +using FluentAssertions; + using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; @@ -59,6 +62,10 @@ public IntegrationTestBase() _testEnvironment = new IntegrationTestEnvironment(); BuildConfiguration = IntegrationTestEnvironment.BuildConfiguration; TempDirectory = new TempDirectory(); + + var drive = new DriveInfo(Directory.GetDirectoryRoot(TempDirectory.Path)); + Console.WriteLine($"Available space for TEMP: {drive.Name} {drive.AvailableFreeSpace / (1024 * 1024)} MB"); + IsCI = IntegrationTestEnvironment.IsCI; } @@ -79,8 +86,11 @@ public IntegrationTestBase() [TestCleanup] public void TempDirectoryCleanup() { - // Delete the directory only when the test succeeded, so we can look at results and logs of failed tests. - if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) + // In CI always delete the results, because we have limited disk space there. + // + // Locally delete the directory only when the test succeeded, so we can look + // at results and logs of failed tests. + if (IsCI || TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) { TempDirectory.Dispose(); } @@ -101,7 +111,17 @@ public static string PrepareArguments(string[] testAssemblies, string testAdapte var arguments = ""; foreach (var path in testAssemblies) { - arguments += path.AddDoubleQuote() + " "; + // The incoming testAssembly path is either a single dll path in quotes or without quotes. + // Or multiple assembly paths in a single string each double quoted and joined by space. + // We trim, and add quotes here to get either: + // C:\1.dll -> "C:\1.dll" + // "C:\1.dll" -> "C:\1.dll" + // "C:\1.dll" "C:\2.dll" -> "C:\1.dll" "C:\2.dll" + // + // For unquoted multi path string C:\1.dll C:\2.dll, we will get "C:\1.dll C:\2.dll" + // which is wrong and will fail later, but it's the test's fault for doing it wrong + // rather than providing an array of strings that this overload takes. + arguments += path.Trim('\"').AddDoubleQuote() + " "; } arguments = arguments.Trim(); @@ -128,6 +148,11 @@ public static string PrepareArguments(string[] testAssemblies, string testAdapte if (!string.IsNullOrWhiteSpace(inIsolation)) { + if (inIsolation != "/InIsolation") + { + // TODO: The whole inIsolation should be just a bool, but it is not, and it's changing in other PR. + throw new InvalidOperationException("InIsolation value must be '/InIsolation'"); + } arguments = string.Concat(arguments, " ", inIsolation); } @@ -160,7 +185,8 @@ public static string PrepareArguments(string testAssembly, string testAdapterPat /// Arguments provided to vstest.console.exe public void InvokeVsTest(string arguments, Dictionary environmentVariables = null) { - ExecuteVsTestConsole(arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, environmentVariables); + var debugEnvironmentVariables = AddDebugEnvironmentVariables(environmentVariables); + ExecuteVsTestConsole(arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, debugEnvironmentVariables); FormatStandardOutCome(); } @@ -170,7 +196,7 @@ public void InvokeVsTest(string arguments, Dictionary environmen /// Arguments provided to vstest.console.exe public void InvokeDotnetTest(string arguments, Dictionary environmentVariables = null) { - environmentVariables ??= new Dictionary(); + var debugEnvironmentVariables = AddDebugEnvironmentVariables(environmentVariables); var vstestConsolePath = GetDotnetRunnerPath(); @@ -181,9 +207,9 @@ public void InvokeDotnetTest(string arguments, Dictionary enviro // This is used in dotnet/sdk to determine path to vstest.console: // https://github.com/dotnet/sdk/blob/main/src/Cli/dotnet/commands/dotnet-test/VSTestForwardingApp.cs#L30-L39 - environmentVariables["VSTEST_CONSOLE_PATH"] = vstestConsolePath; + debugEnvironmentVariables["VSTEST_CONSOLE_PATH"] = vstestConsolePath; - ExecutePatchedDotnet("test", arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, environmentVariables); + ExecutePatchedDotnet("test", arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, debugEnvironmentVariables); FormatStandardOutCome(); } @@ -201,10 +227,39 @@ public void InvokeVsTestForExecution(string testAssembly, Dictionary environmentVariables = null) { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); - InvokeVsTest(arguments, environmentVariables); } + private Dictionary AddDebugEnvironmentVariables(Dictionary environmentVariables) + { + environmentVariables ??= new Dictionary(); + + if (_testEnvironment.DebugInfo != null) + { + if (_testEnvironment.DebugInfo.DebugVSTestConsole) + { + environmentVariables["VSTEST_RUNNER_DEBUG_ATTACHVS"] = "1"; + } + + if (_testEnvironment.DebugInfo.DebugTestHost) + { + environmentVariables["VSTEST_HOST_DEBUG_ATTACHVS"] = "1"; + } + + if (_testEnvironment.DebugInfo.DebugDataCollector) + { + environmentVariables["VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS"] = "1"; + } + + if (_testEnvironment.DebugInfo.NoDefaultBreakpoints) + { + environmentVariables["VSTEST_DEBUG_NOBP"] = "1"; + } + } + + return environmentVariables; + } + /// /// Invokes vstest.console to discover tests in a test assembly. "/listTests" is appended to the arguments. /// @@ -235,12 +290,19 @@ public void ExecuteNotSupportedRunnerFrameworkTests(string runnerFramework, stri /// /// Validate if the overall test count and results are matching. /// - /// Passed test count - /// Failed test count - /// Skipped test count - public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, int skippedTestsCount) + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatus(int passed, int failed, int skipped) { - var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; + // TODO: Switch on the actual version of vstest console when we have that set on test environment. + if (_testEnvironment.VSTestConsoleInfo != null && _testEnvironment.VSTestConsoleInfo.Path.Contains($"{Path.DirectorySeparatorChar}15.")) + { + ValidateSummaryStatusv15(passed, failed, skipped); + return; + } + + var totalTestCount = passed + failed + skipped; if (totalTestCount == 0) { // No test should be found/run @@ -262,19 +324,70 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in else { var summaryStatus = string.Format(TotalTestsMessage, totalTestCount); - if (passedTestsCount != 0) + if (passed != 0) { - summaryStatus += string.Format(PassedTestsMessage, passedTestsCount); + summaryStatus += string.Format(PassedTestsMessage, passed); } - if (failedTestsCount != 0) + if (failed != 0) { - summaryStatus += string.Format(FailedTestsMessage, failedTestsCount); + summaryStatus += string.Format(FailedTestsMessage, failed); } - if (skippedTestsCount != 0) + if (skipped != 0) { - summaryStatus += string.Format(SkippedTestsMessage, skippedTestsCount); + summaryStatus += string.Format(SkippedTestsMessage, skipped); + } + + Assert.IsTrue( + _standardTestOutput.Contains(summaryStatus), + "The Test summary does not match.{3}Expected summary: {1}{3}Test Output: {0}{3}Standard Error: {2}{3}Arguments: {4}{3}", + _standardTestOutput, + summaryStatus, + _standardTestError, + Environment.NewLine, + _arguments); + } + } + + /// + /// Validate if the overall test count and results are matching. + /// + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatusv15(int passed, int failed, int skipped) + { + // example: Total tests: 6. Passed: 2. Failed: 2. Skipped: 2. + var totalTestCount = passed + failed + skipped; + if (totalTestCount == 0) + { + // No test should be found/run + StringAssert.DoesNotMatch( + _standardTestOutput, + new Regex("Total tests\\:"), + "Excepted: There should not be test summary{2}Actual: {0}{2}Standard Error: {1}{2}Arguments: {3}{2}", + _standardTestOutput, + _standardTestError, + Environment.NewLine, + _arguments); + } + else + { + var summaryStatus = $"Total tests: {totalTestCount}."; + if (passed != 0) + { + summaryStatus += $" Passed: {passed}."; + } + + if (failed != 0) + { + summaryStatus += $" Failed: {failed}."; + } + + if (skipped != 0) + { + summaryStatus += $" Skipped: {skipped}."; } Assert.IsTrue( @@ -427,11 +540,22 @@ protected string GetAssetFullPath(string assetName) return _testEnvironment.GetTestAsset(assetName); } - protected string GetAssetFullPath(string assetName, string targetFramework) + protected string GetTestDllForFramework(string assetName, string targetFramework) { return _testEnvironment.GetTestAsset(assetName, targetFramework); } + protected List GetTestDlls(params string[] assetNames) + { + var assets = new List(); + foreach (var assetName in assetNames) + { + assets.Add(GetAssetFullPath(assetName)); + } + + return assets; + } + protected string GetProjectFullPath(string projectName) { return _testEnvironment.GetTestProject(projectName); @@ -478,7 +602,7 @@ protected bool IsNetCoreRunner() } /// - /// Gets the path to vstest.console.exe. + /// Gets the path to vstest.console.exe or dotnet.exe. /// /// /// Full path to test runner @@ -489,7 +613,14 @@ public virtual string GetConsoleRunnerPath() if (IsDesktopRunner()) { - consoleRunnerPath = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.exe"); + if (!string.IsNullOrWhiteSpace(_testEnvironment.VSTestConsoleInfo?.Path)) + { + consoleRunnerPath = _testEnvironment.VSTestConsoleInfo.Path; + } + else + { + consoleRunnerPath = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.exe"); + } } else if (IsNetCoreRunner()) { @@ -501,13 +632,13 @@ public virtual string GetConsoleRunnerPath() Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); } - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: \"{0}\"", consoleRunnerPath); return consoleRunnerPath; } protected virtual string SetVSTestConsoleDLLPathInArgs(string args) { - var vstestConsoleDll = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + var vstestConsoleDll = GetDotnetRunnerPath(); vstestConsoleDll = vstestConsoleDll.AddDoubleQuote(); args = string.Concat( vstestConsoleDll, @@ -516,50 +647,35 @@ protected virtual string SetVSTestConsoleDLLPathInArgs(string args) return args; } - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper() - { - return GetVsTestConsoleWrapper(TempDirectory); - } - - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(out TempDirectory logFileDir) - { - logFileDir = new TempDirectory(); - return GetVsTestConsoleWrapper(logFileDir); - } /// /// Returns the VsTestConsole Wrapper. /// /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, TraceLevel traceLevel = TraceLevel.Verbose) + public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TraceLevel traceLevel = TraceLevel.Verbose) { - string logFilePath = null; + ConsoleParameters consoleParameters = new(); if (traceLevel != TraceLevel.Off) { - if (!Directory.Exists(logFileDir.Path)) + if (!Directory.Exists(TempDirectory.Path)) { - Directory.CreateDirectory(logFileDir.Path); + Directory.CreateDirectory(TempDirectory.Path); } // Directory is already unique so there is no need to have a unique file name. - logFilePath = Path.Combine(logFileDir.Path, "log.txt"); + var logFilePath = Path.Combine(TempDirectory.Path, "log.txt"); if (!File.Exists(logFilePath)) { File.Create(logFilePath).Close(); } Console.WriteLine($"Logging diagnostics in {logFilePath}"); + consoleParameters.LogFilePath = logFilePath; } var consoleRunnerPath = IsNetCoreRunner() - ? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") - : GetConsoleRunnerPath(); + ? GetDotnetRunnerPath() + : GetConsoleRunnerPath(); var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; var dotnetPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); @@ -568,8 +684,39 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, T throw new FileNotFoundException($"File '{dotnetPath}' was not found."); } - ConsoleParameters consoleParameters = traceLevel == TraceLevel.Off ? new() : new() { LogFilePath = logFilePath }; - var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, consoleParameters); + if (!File.Exists(consoleRunnerPath)) + { + throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found."); + } + + Console.WriteLine($"Console runner path: {consoleRunnerPath}"); + + VsTestConsoleWrapper vstestConsoleWrapper; + + // Providing any environment variable to vstest.console will clear all existing environment variables, + // this works around it by copying all existing variables, and adding debug. But we only want to do that + // when we are setting any debug variables. + // TODO: This is scheduled to be fixed in 17.3, where it will start working normally. We will just add those + // variables, unless we explicitly say to clean them. https://github.com/microsoft/vstest/pull/3433 + // Remove this code later, and just pass the variables you want to add. + var debugEnvironmentVariables = AddDebugEnvironmentVariables(new Dictionary()); + Dictionary environmentVariables = new(); + if (debugEnvironmentVariables.Count > 0) + { + Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); + foreach (var pair in debugEnvironmentVariables) + { + environmentVariables[pair.Key] = pair.Value; + } + } + + if (environmentVariables.Count > 0) + { + // This clears all variables, so we copy all environment variables, and add the debug ones to them. + consoleParameters.EnvironmentVariables = environmentVariables; + } + + vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, consoleParameters); vstestConsoleWrapper.StartSession(); return vstestConsoleWrapper; @@ -772,13 +919,8 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - var assertFullPaths = new string[assetNames.Length]; - for (var i = 0; i < assetNames.Length; i++) - { - assertFullPaths[i] = GetAssetFullPath(assetNames[i]).AddDoubleQuote(); - } - - return string.Join(" ", assertFullPaths); + // Double quoted sources sepearated by space. + return string.Join(" ", GetTestDlls(assetNames).Select(a => a.AddDoubleQuote())); } protected static string GetDiagArg(string rootDir) @@ -817,5 +959,15 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) return path; } - protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); + protected string GetDotnetRunnerPath() => _testEnvironment.VSTestConsoleInfo?.Path ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + + protected void StdOutHasNoWarnings() + { + StdOut.Should().NotContainEquivalentOf("warning"); + } + + protected void StdErrHasTestRunFailedMessageButNoOtherError() + { + StdErr?.Trim().Should().Be("Test Run Failed."); + } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 2de0b4dd03..01c3069ee1 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -193,6 +193,9 @@ public string RunnerFramework // A known AzureDevOps env variable meaning we are running in CI. public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; + public DebugInfo DebugInfo { get; set; } + public VSTestConsoleInfo VSTestConsoleInfo { get; set; } + public List DllInfos { get; set; } = new(); /// /// Gets the full path to a test asset. @@ -235,8 +238,19 @@ public string GetTestAsset(string assetName, string targetFramework) targetFramework, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + // Update the path to be taken from the compatibility matrix instead of from the root folder. + if (DllInfos.Count > 0) + { + foreach (var dllInfo in DllInfos) + { + assetPath = dllInfo.UpdatePath(assetPath); + } + } + + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); + // If you are thinking about wrapping the path in double quotes here, + // then don't. File.Exist cannot handle quoted paths, and we use it in a lot of places. return assetPath; } @@ -310,7 +324,7 @@ public string GetTestProject(string assetName) simpleAssetName, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\".", assetPath); return assetPath; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index 9e010ea13c..527a9a44b3 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -25,6 +25,7 @@ + diff --git a/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs new file mode 100644 index 0000000000..00deda8823 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer +public class NetTestSdkInfo : DllInfo +{ + public NetTestSdkInfo() + { + Name = "Testhost"; + PropertyName = "VSTestConsole"; + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs new file mode 100644 index 0000000000..3ba8deabc9 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer +public class VSTestConsoleInfo +{ + public string? VersionType { get; set; } + public string? Version { get; set; } + public string? Path { get; set; } + + public override string ToString() => $" vstest.console = {Version} [{VersionType}]"; +} + diff --git a/test/TestAssets/MSTestProject1/MSTestProject1.csproj b/test/TestAssets/MSTestProject1/MSTestProject1.csproj new file mode 100644 index 0000000000..6e15698454 --- /dev/null +++ b/test/TestAssets/MSTestProject1/MSTestProject1.csproj @@ -0,0 +1,28 @@ + + + + + + netcoreapp2.1;net451 + netcoreapp3.1 + false + Preview + + + + + $(MSTestFrameworkVersion) + + + $(MSTestAdapterVersion) + + + $(NETTestSdkVersion) + + + + + + + + diff --git a/test/TestAssets/MSTestProject1/UnitTest1.cs b/test/TestAssets/MSTestProject1/UnitTest1.cs new file mode 100644 index 0000000000..17c1760f18 --- /dev/null +++ b/test/TestAssets/MSTestProject1/UnitTest1.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestProject1; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + Assert.AreEqual(2, 2); + } + + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(2, 3); + } + + [Ignore] + [TestMethod] + public void SkippedTest() + { + } +} diff --git a/test/TestAssets/MSTestProject2/MSTestProject2.csproj b/test/TestAssets/MSTestProject2/MSTestProject2.csproj new file mode 100644 index 0000000000..6e15698454 --- /dev/null +++ b/test/TestAssets/MSTestProject2/MSTestProject2.csproj @@ -0,0 +1,28 @@ + + + + + + netcoreapp2.1;net451 + netcoreapp3.1 + false + Preview + + + + + $(MSTestFrameworkVersion) + + + $(MSTestAdapterVersion) + + + $(NETTestSdkVersion) + + + + + + + + diff --git a/test/TestAssets/MSTestProject2/UnitTest1.cs b/test/TestAssets/MSTestProject2/UnitTest1.cs new file mode 100644 index 0000000000..1f6d2bf47e --- /dev/null +++ b/test/TestAssets/MSTestProject2/UnitTest1.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestProject2; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + Assert.AreEqual(2, 2); + } + + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(2, 3); + } + + [Ignore] + [TestMethod] + public void SkippedTest() + { + } +} diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index 71bdd890cd..e349d2626f 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -114,6 +114,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "XUnit100Passing", "performa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "XUnit1000Passing", "performance\XUnit1000Passing\XUnit1000Passing.csproj", "{15DC879D-1A60-4745-8DF4-EBC36EE12339}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "basic", "basic", "{2633D125-64A7-456C-AD37-F8A6B56C2403}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -748,6 +756,42 @@ Global {15DC879D-1A60-4745-8DF4-EBC36EE12339}.Release|x64.Build.0 = Release|Any CPU {15DC879D-1A60-4745-8DF4-EBC36EE12339}.Release|x86.ActiveCfg = Release|Any CPU {15DC879D-1A60-4745-8DF4-EBC36EE12339}.Release|x86.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x64.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x64.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x86.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x86.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|Any CPU.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x64.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x64.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x86.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x86.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|Any CPU.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -766,6 +810,10 @@ Global {06724523-611A-4054-A1AC-29DA2E87DD87} = {0C9CA869-32FD-4A9E-8885-E2E19786C746} {15DC879D-1A60-4745-8DF4-EBC36EE12339} = {0C9CA869-32FD-4A9E-8885-E2E19786C746} EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {10AA955C-B412-41A8-899F-8609AAE19F61} = {2633D125-64A7-456C-AD37-F8A6B56C2403} + {E166D337-4033-4209-863F-8F77675EAEE8} = {2633D125-64A7-456C-AD37-F8A6B56C2403} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D2334DAA-F7B2-450E-ABA4-FBC185152500} EndGlobalSection diff --git a/test/TestAssets/Tools/Program.cs b/test/TestAssets/Tools/Program.cs new file mode 100644 index 0000000000..94783ab8a9 --- /dev/null +++ b/test/TestAssets/Tools/Program.cs @@ -0,0 +1,7 @@ +// See https://aka.ms/new-console-template for more information +using System; + +// This project is used to restore TestPlatform and TestPlatform.CLI tools packages +// for testing with older versions. + +Console.WriteLine("Hello, World!"); diff --git a/test/TestAssets/Tools/Tools.csproj b/test/TestAssets/Tools/Tools.csproj new file mode 100644 index 0000000000..5981f1c93c --- /dev/null +++ b/test/TestAssets/Tools/Tools.csproj @@ -0,0 +1,23 @@ + + + + ..\..\..\ + true + true + + + + + + net5.0 + Exe + hanging_child + + + + + + + + + diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj index 793d988e7d..435b57d4fd 100644 --- a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -16,7 +16,7 @@ Exe - +