Modernize the performance scripts - add the ability to test a locally built NuGet

scripts/perftests/PerformanceTestRunner.ps1

@@ -20,8 +20,19 @@ How many times to run each test. The default is 3
 .PARAMETER skipRepoCleanup
 Whether to delete the checked out repos from the test cases.
 
+.PARAMETER additionalOptions
+Additional options such as ones to skip certain runs like -skipWarmup or maybe run the local version of NuGet, -useLocallyBuiltNuGet. 
+To get the list of all options check out RunPerformanceTests.ps1 in the same folder.
+
 .EXAMPLE
 .\PerformanceTestRunner.ps1 -resultsFolderPath resultsFolder -nugetClientFilePaths F:\NuGetExe\NuGet.exe,"C:\Program Files\dotnet\dotnet.exe" 
+
+.EXAMPLE
+.\PerformanceTestRunner.ps1 -resultsFolderPath resultsFolder -nugetClientFilePaths "C:\Program Files\Microsoft Visual Studio\2022\Preview\MSBuild\Current\Bin\amd64\MSBuild.exe" -skipRepoCleanup -additionalOptions "-useLocallyBuiltNuGet -staticGraphRestore"
+
+.EXAMPLE
+.\PerformanceTestRunner.ps1 -resultsFolderPath resultsFolder -nugetClientFilePaths "C:\Program Files\Microsoft Visual Studio\2022\Preview\MSBuild\Current\Bin\amd64\MSBuild.exe" -skipRepoCleanup -additionalOptions "-useLocallyBuiltNuGet"
+
 #>
 Param(
     [Parameter(Mandatory = $true)]
@@ -31,7 +42,9 @@ Param(
     [string] $testRootFolderPath,
     [string] $logsFolderPath,
     [int] $iterationCount = 3,
-    [switch] $skipRepoCleanup
+    [string] $testCaseDirectory,
+    [switch] $skipRepoCleanup,
+    [string] $additionalOptions
 )
 
 . "$PSScriptRoot\PerformanceTestUtilities.ps1"
@@ -86,10 +99,16 @@ Try
                 Exit 1
             }
 
-            Log "Discovering the test cases."
-            $testFiles = $(Get-ChildItem $PSScriptRoot\testCases "Test-*.ps1" ) | ForEach-Object { $_.FullName }
-            Log "Discovered test cases: $testFiles" "green"
+            $testCasesDirectory = "$PSScriptRoot\testCases"
+            If (-not [string]::IsNullOrWhiteSpace($testCaseDirectory))
+            {
+                $testCasesDirectory = $testCaseDirectory
+            }
 
+            Log "Discovering the test cases in $testCasesDirectory"
+            $testFiles = $(Get-ChildItem $testCasesDirectory "Test-*.ps1" ) | ForEach-Object { $_.FullName }
+            Log "Discovered test cases: $testFiles" "green"
+
             $testFiles | ForEach-Object {
                 $testCase = $_
                 Try
@@ -100,7 +119,8 @@ Try
                         -resultsFolderPath $resultsFolderPath `
                         -logsFolderPath $logsFolderPath `
                         -nugetFoldersPath $nugetFoldersPath `
-                        -iterationCount $iterationCount
+                        -iterationCount $iterationCount `
+                        -additionalOptions $additionalOptions
                 }
                 Catch
                 {

scripts/perftests/PerformanceTestUtilities.ps1

@@ -102,12 +102,13 @@ Function DownloadRepository([string] $repository, [string] $commitHash, [string]
 {
     If (Test-Path $sourceFolderPath)
     {
-        Log "Skipping the cloning of $repository as $sourceFolderPath is not empty" -color "Yellow"
+        Log "Skipping the cloning of $repository as $sourceFolderPath is not empty. Running git clean" -color "Yellow"
+        git -C $sourceFolderPath clean -xdf | out-null
     }
     Else
     {
         git clone $repository $sourceFolderPath
-        git -C $sourceFolderPath checkout $commitHash
+        git -C $sourceFolderPath checkout $commitHash | out-null
     }
 }
 
@@ -124,7 +125,7 @@ Function GetSolutionFilePath([string] $repository, [string] $sourceFolderPath)
     }
     Else
     {
-        $possibleSln = Get-ChildItem $sourceFolderPath *.sln
+        $possibleSln = Get-ChildItem $sourceFolderPath *.sln  | Where-Object{$_.FullName -notlike '*.slnf'}
         If ($possibleSln.Length -eq 0)
         {
             Log "No solution files found in $sourceFolderPath" "red"
@@ -180,7 +181,7 @@ Function GetClientVersion([string] $nugetClientFilePath)
     ElseIf($(IsClientMSBuildExe $nugetClientFilePath))
     {
         $clientDir = Split-Path -Path $nugetClientFilePath
-        $nugetClientPath = Resolve-Path (Join-Path -Path $clientDir -ChildPath "../../../Common7/IDE/CommonExtensions/Microsoft/NuGet/NuGet.Build.Tasks.dll")
+        $nugetClientPath = Resolve-Path (Join-Path -Path $clientDir -ChildPath "../../../../Common7/IDE/CommonExtensions/Microsoft/NuGet/NuGet.Build.Tasks.dll")
         $versionInfo = Get-ChildItem $nugetClientPath | % versioninfo | Select-Object FileVersion
         Return $(($versionInfo -split '\n')[0]).TrimStart("@{").TrimEnd('}').Substring("FileVersion=".Length)
     }
@@ -224,7 +225,7 @@ Function SetupNuGetFolders([string] $nugetClientFilePath, [string] $nugetFolders
 # This should only be invoked by the the performance tests
 Function CleanNuGetFolders([string] $nugetClientFilePath, [string] $nugetFoldersPath)
 {
-    Log "Cleanup up the NuGet folders - global packages folder, http/plugins caches. Client: $nugetClientFilePath. Folders: $nugetFoldersPath"
+    Log "Cleanup the NuGet folders - global packages folder, http/plugins caches. Client: $nugetClientFilePath. Folders: $nugetFoldersPath"
 
     LocalsClearAll $nugetClientFilePath
 
@@ -242,25 +243,20 @@ Function CleanNuGetFolders([string] $nugetClientFilePath, [string] $nugetFolders
 Function RunPerformanceTestsOnGitRepository(
     [string] $nugetClientFilePath,
     [string] $sourceRootFolderPath,
-    [string] $testCaseName,
     [string] $repoUrl,
     [string] $commitHash,
-    [string] $resultsFilePath,
+    [string] $resultsFolderPath,
     [string] $nugetFoldersPath,
     [string] $logsFolderPath,
     [int] $iterationCount,
-    [switch] $staticGraphRestore)
+    [string] $additionalOptions)
 {
+    $testCaseName = GenerateNameFromGitUrl $repoUrl
+    $resultsFilePath = [System.IO.Path]::Combine($resultsFolderPath, "$testCaseName.csv")
     $solutionFilePath = SetupGitRepository -repository $repoUrl -commitHash $commitHash -sourceFolderPath $([System.IO.Path]::Combine($sourceRootFolderPath, $testCaseName))
+    $sb = [scriptblock]::Create("$PSScriptRoot\RunPerformanceTests.ps1 -nugetClientFilePath ""$nugetClientFilePath"" -solutionFilePath $solutionFilePath -resultsFilePath $resultsFilePath -logsFolderPath $logsFolderPath -nugetFoldersPath $nugetFoldersPath -iterationCount $iterationCount " + $additionalOptions)
     SetupNuGetFolders $nugetClientFilePath $nugetFoldersPath
-    . "$PSScriptRoot\RunPerformanceTests.ps1" `
-        -nugetClientFilePath $nugetClientFilePath `
-        -solutionFilePath $solutionFilePath `
-        -resultsFilePath $resultsFilePath `
-        -logsFolderPath $logsFolderPath `
-        -nugetFoldersPath $nugetFoldersPath `
-        -iterationCount $iterationCount `
-        -staticGraphRestore:$staticGraphRestore
+    & $sb
 }
 
 Function GetProcessorInfo()
@@ -350,7 +346,10 @@ Function RunRestore(
     [switch] $cleanPluginsCache,
     [switch] $killMsBuildAndDotnetExeProcesses,
     [switch] $force,
-    [switch] $staticGraphRestore)
+    [switch] $staticGraphRestore,
+    [switch] $cleanRepository,
+    [switch] $useLocallyBuiltNuGet,
+    [switch] $forceLegacyResolverFallback)
 {
     $isClientDotnetExe = IsClientDotnetExe $nugetClientFilePath
     $isClientMSBuild = IsClientMSBuildExe $nugetClientFilePath
@@ -362,7 +361,7 @@ Function RunRestore(
         Return
     }
 
-    Log "Running $nugetClientFilePath restore with cleanGlobalPackagesFolder:$cleanGlobalPackagesFolder cleanHttpCache:$cleanHttpCache cleanPluginsCache:$cleanPluginsCache killMsBuildAndDotnetExeProcesses:$killMsBuildAndDotnetExeProcesses force:$force"
+    Log "Running $nugetClientFilePath restore with cleanGlobalPackagesFolder:$cleanGlobalPackagesFolder cleanHttpCache:$cleanHttpCache cleanPluginsCache:$cleanPluginsCache killMsBuildAndDotnetExeProcesses:$killMsBuildAndDotnetExeProcesses force:$force staticGraphRestore:$staticGraphRestore cleanRepository:$cleanRepository"
 
     $solutionPackagesFolderPath = $Env:NUGET_SOLUTION_PACKAGES_FOLDER_PATH
 
@@ -410,6 +409,12 @@ Function RunRestore(
         }
     }
 
+    if($cleanRepository)
+    {
+        $repositoryPath = [System.IO.Path]::GetDirectoryName($solutionFilePath)
+        git -C $repositoryPath clean -xdf | out-null
+    }
+
     if($killMsBuildAndDotnetExeProcesses)
     {
         Stop-Process -name msbuild*,dotnet* -Force
@@ -479,9 +484,44 @@ Function RunRestore(
         $staticGraphOutputValue = "N/A"
     }
 
-    $stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
+    If($isClientDotnetExe -Or $isClientMSBuild)
+    {
+        # Always disable NuGetAudit. It's something impacted by the time of execution, and not helpful in these type of performance tests.
+        $arguments.Add("/p:NuGetAudit=false")   
+    }
+
+    if(($isClientDotnetExe -Or $isClientMSBuild) -And $forceLegacyResolverFallback)
+    {
+        $arguments.Add("/p:RestoreUseLegacyDependencyResolver=true")
+    }
 
+    if($useLocallyBuiltNuGet)
+    {
+        if($isClientMSBuild)
+        {
+            $NETFramework = "net472"
+            $Configuration = "release"
+            $packDllPath = Join-Path $NuGetClientRoot "artifacts\NuGet.Build.Tasks.Pack\bin\$Configuration\$NETFramework\NuGet.Build.Tasks.Pack.dll"
+            $packTargetsPath = Join-Path $NuGetClientRoot "artifacts\NuGet.Build.Tasks.Pack\bin\$Configuration\$NETFramework\NuGet.Build.Tasks.Pack.targets"
+            $nugetRestoreTargetsPath = Join-Path $NuGetClientRoot "artifacts\NuGet.Build.Tasks\bin\$Configuration\$NETFramework\NuGet.targets"
+            $nugetPropsPath = Join-Path $NuGetClientRoot "artifacts\NuGet.Build.Tasks\bin\$Configuration\$NETFramework\NuGet.props"
+            $consoleExePath = Join-Path $NuGetClientRoot "artifacts\NuGet.Build.Tasks.Console\bin\$Configuration\$NETFramework\NuGet.Build.Tasks.Console.exe"            
+            $arguments.Add("/p:NuGetRestoreTargets=$nugetRestoreTargetsPath");
+            $arguments.Add("/p:NuGetPropsFile=$nugetPropsPath");
+            $arguments.Add("/p:NuGetBuildTasksPackTargets=$packTargetsPath");
+            $arguments.Add("/p:NuGetConsoleProcessFileName=$consoleExePath");
+            $arguments.Add("/p:ImportNuGetBuildTasksPackTargetsFromSdk=true");
+            $arguments.Add("/p:NuGetPackTaskAssemblyFile=$packDllPath");
+        } 
+        else 
+        {
+            Log "Locally built NuGet can only be used with msbuild.exe" "red"
+        }
+    }
+
+    $stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
     $logs = . $nugetClientFilePath $arguments | Out-String
+
     if($LASTEXITCODE -ne 0)
     { 
         throw "The command `"$nugetClientFilePath $arguments`" finished with exit code $LASTEXITCODE.`n" + $logs
@@ -516,20 +556,20 @@ Function RunRestore(
 
     If (!(Test-Path $resultsFilePath))
     {
-        $columnHeaders = "Client Name,Client Version,Solution Name,Test Run ID,Scenario Name,Total Time (seconds),Core Restore Time (seconds),Force,Static Graph," + `
+        $columnHeaders = "Client Name,Client Version,Locally Built NuGet,Solution Name,Test Run ID,Scenario Name,Total Time (seconds),Core Restore Time (seconds),Force,Static Graph," + `
             "Global Packages Folder .nupkg Count,Global Packages Folder .nupkg Size (MB),Global Packages Folder File Count,Global Packages Folder File Size (MB),Clean Global Packages Folder," + `
-            "HTTP Cache File Count,HTTP Cache File Size (MB),Clean HTTP Cache,Plugins Cache File Count,Plugins Cache File Size (MB),Clean Plugins Cache,Kill MSBuild and dotnet Processes," + `
+            "HTTP Cache File Count,HTTP Cache File Size (MB),Clean HTTP Cache,Plugins Cache File Count,Plugins Cache File Size (MB),Clean Plugins Cache,Kill MSBuild and dotnet Processes,Clean git repo,Force Legacy resolver," + `
             "Processor Name,Processor Physical Core Count,Processor Logical Core Count"
 
         OutFileWithCreateFolders $resultsFilePath $columnHeaders
     }
 
-    $data = "$clientName,$clientVersion,$solutionName,$testRunId,$scenarioName,$totalTime,$restoreCoreTime,$force,$staticGraphOutputValue," + `
+    $data = "$clientName,$clientVersion,$useLocallyBuiltNuGet,$solutionName,$testRunId,$scenarioName,$totalTime,$restoreCoreTime,$force,$staticGraphOutputValue," + `
         "$($globalPackagesFolderNupkgFilesInfo.Count),$($globalPackagesFolderNupkgFilesInfo.TotalSizeInMB),$($globalPackagesFolderFilesInfo.Count),$($globalPackagesFolderFilesInfo.TotalSizeInMB),$cleanGlobalPackagesFolder," + `
-        "$($httpCacheFilesInfo.Count),$($httpCacheFilesInfo.TotalSizeInMB),$cleanHttpCache,$($pluginsCacheFilesInfo.Count),$($pluginsCacheFilesInfo.TotalSizeInMB),$cleanPluginsCache,$killMsBuildAndDotnetExeProcesses," + `
+        "$($httpCacheFilesInfo.Count),$($httpCacheFilesInfo.TotalSizeInMB),$cleanHttpCache,$($pluginsCacheFilesInfo.Count),$($pluginsCacheFilesInfo.TotalSizeInMB),$cleanPluginsCache,$killMsBuildAndDotnetExeProcesses,$cleanRepository,$forceLegacyResolverFallback," + `
         "$($processorInfo.Name),$($processorInfo.NumberOfCores),$($processorInfo.NumberOfLogicalProcessors)"
 
     Add-Content -Path $resultsFilePath -Value $data
 
-    Log "Finished measuring."
+    Log "Finished measuring in $totalTime sec"
 }

scripts/perftests/README.md

@@ -13,6 +13,7 @@ The scenarios are sequential as follows:
 
 1. Clean restore - no http cache & other local caches, no files in the global package folder, absolutely everything gets downloaded and extracted.
 1. Cold restore - There is only an http cache. This tells us more about the installation/extraction time. Potentially we might see some extra http calls depending on the project graph.
+1. Offline restore - no http cache or plugin cache, but there is a global packages folder. This simulates setups where the gpf is shared across different machines. Ideally these restores are offline, and do not make any remote calls.
 1. Force restore - The http cache & global packages folder are full. This usually means that there are no package downloads or installations happening.
 1. NoOp restore
 
@@ -25,3 +26,9 @@ The `PerformanceTestUtilities.ps1` script is a collection of utility functions t
 Note that it's very important to initialize the script from a location that does not have a global.json in it's directory path. This can skew the results if you are dealing with SDK based projects.
 
 To run either the performance tests or the runner, run `Get-Help scriptName.ps1` and/or `Get-Help scriptName.ps1 -examples`
+
+### Using the scripts to test local builds
+
+- The way NuGet is integrated within MSBuild, allows us to easily bootstrap the restore code with MSBuild and run the current locally built bits.
+To do that, simply pass `-additionalOptions "-useLocallyBuiltNuGet"` in the PerformanceTestRunner as the scripts' example suggests.
+Other notable features include toggling between the default and legacy resolver `-forceLegacyResolverFallback` or opting into graph restore `-staticGraphRestore`.