From 4a438cde67a1509ec2e7a2b72ec026aa5e0c7bc2 Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Thu, 16 Jun 2022 22:27:33 -0700 Subject: [PATCH 01/14] Update nuget client dependency packages --- src/code/PowerShellGet.csproj | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/code/PowerShellGet.csproj b/src/code/PowerShellGet.csproj index 6d580abac..dbd51ee95 100644 --- a/src/code/PowerShellGet.csproj +++ b/src/code/PowerShellGet.csproj @@ -15,12 +15,12 @@ - - - - - - + + + + + + From a021992d8e1e4d0524773efcf3259be60dd0b3b1 Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 29 Jun 2022 19:38:00 -0700 Subject: [PATCH 02/14] Add benchmark project --- test/perf/benchmarks/Benchmarks.cs | 25 ++++++++++++++++++ test/perf/benchmarks/Program.cs | 12 +++++++++ test/perf/benchmarks/benchmarks.csproj | 36 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 test/perf/benchmarks/Benchmarks.cs create mode 100644 test/perf/benchmarks/Program.cs create mode 100644 test/perf/benchmarks/benchmarks.csproj diff --git a/test/perf/benchmarks/Benchmarks.cs b/test/perf/benchmarks/Benchmarks.cs new file mode 100644 index 000000000..ed27bd513 --- /dev/null +++ b/test/perf/benchmarks/Benchmarks.cs @@ -0,0 +1,25 @@ +using System; +using BenchmarkDotNet; +using BenchmarkDotNet.Attributes; + +namespace benchmarks +{ + internal class Benchmarks + { + [Benchmark] + public PSResourceInfo Scenario1() + { + PSResourceInfo a = null; + + // Implement your benchmark here + + return a; + } + + [Benchmark] + public void Scenario2() + { + // Implement your benchmark here + } + } +} diff --git a/test/perf/benchmarks/Program.cs b/test/perf/benchmarks/Program.cs new file mode 100644 index 000000000..a73026086 --- /dev/null +++ b/test/perf/benchmarks/Program.cs @@ -0,0 +1,12 @@ +using BenchmarkDotNet.Running; + +namespace benchmarks +{ + public sealed class Program + { + public static void Main(string[] args) + { + var summary = BenchmarkRunner.Run(); + } + } +} \ No newline at end of file diff --git a/test/perf/benchmarks/benchmarks.csproj b/test/perf/benchmarks/benchmarks.csproj new file mode 100644 index 000000000..bd4f1a490 --- /dev/null +++ b/test/perf/benchmarks/benchmarks.csproj @@ -0,0 +1,36 @@ + + + net5.0 + Exe + + + AnyCPU + pdbonly + true + true + true + Release + false + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From a633df8c73242570f2de2c8256d6e23f34d7db2d Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Thu, 30 Jun 2022 19:00:09 -0700 Subject: [PATCH 03/14] target netstandard2.0 --- test/perf/benchmarks/benchmarks.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/benchmarks/benchmarks.csproj b/test/perf/benchmarks/benchmarks.csproj index bd4f1a490..7f5e03c1e 100644 --- a/test/perf/benchmarks/benchmarks.csproj +++ b/test/perf/benchmarks/benchmarks.csproj @@ -1,6 +1,6 @@ - net5.0 + netstandard2.0 Exe From e4d179f84f0d5b8f7d5848049b39ebd8169faa56 Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Thu, 30 Jun 2022 20:28:48 -0700 Subject: [PATCH 04/14] Update target framework and get 'run' working --- src/code/PowerShellGet.csproj | 7 +++++++ test/perf/benchmarks/Benchmarks.cs | 8 ++++---- test/perf/benchmarks/benchmarks.csproj | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/code/PowerShellGet.csproj b/src/code/PowerShellGet.csproj index dbd51ee95..34478ecc4 100644 --- a/src/code/PowerShellGet.csproj +++ b/src/code/PowerShellGet.csproj @@ -27,6 +27,13 @@ + + + + + + + diff --git a/test/perf/benchmarks/Benchmarks.cs b/test/perf/benchmarks/Benchmarks.cs index ed27bd513..e0ff37018 100644 --- a/test/perf/benchmarks/Benchmarks.cs +++ b/test/perf/benchmarks/Benchmarks.cs @@ -4,16 +4,16 @@ namespace benchmarks { - internal class Benchmarks + public class Benchmarks { [Benchmark] - public PSResourceInfo Scenario1() + public void Scenario1() { - PSResourceInfo a = null; + //PSResourceInfo a = null; // Implement your benchmark here - return a; + //return a; } [Benchmark] diff --git a/test/perf/benchmarks/benchmarks.csproj b/test/perf/benchmarks/benchmarks.csproj index 7f5e03c1e..1dc8da747 100644 --- a/test/perf/benchmarks/benchmarks.csproj +++ b/test/perf/benchmarks/benchmarks.csproj @@ -1,6 +1,6 @@ - netstandard2.0 + net6.0 Exe From da816e9a40da4b2649f941ec77a5368bb51c000a Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 6 Jul 2022 10:27:52 -0700 Subject: [PATCH 05/14] Finish writing tests for remote repositories --- src/code/AssemblyInfo.cs | 6 +++ src/code/PowerShellGet.csproj | 5 +- test/perf/benchmarks/Benchmarks.cs | 25 --------- test/perf/benchmarks/BenchmarksV2.cs | 64 +++++++++++++++++++++++ test/perf/benchmarks/BenchmarksV3.cs | 71 ++++++++++++++++++++++++++ test/perf/benchmarks/Program.cs | 6 ++- test/perf/benchmarks/benchmarks.csproj | 13 +---- 7 files changed, 149 insertions(+), 41 deletions(-) create mode 100644 src/code/AssemblyInfo.cs delete mode 100644 test/perf/benchmarks/Benchmarks.cs create mode 100644 test/perf/benchmarks/BenchmarksV2.cs create mode 100644 test/perf/benchmarks/BenchmarksV3.cs diff --git a/src/code/AssemblyInfo.cs b/src/code/AssemblyInfo.cs new file mode 100644 index 000000000..95fcac8d6 --- /dev/null +++ b/src/code/AssemblyInfo.cs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("benchmarks")] \ No newline at end of file diff --git a/src/code/PowerShellGet.csproj b/src/code/PowerShellGet.csproj index 34478ecc4..c40f72d0f 100644 --- a/src/code/PowerShellGet.csproj +++ b/src/code/PowerShellGet.csproj @@ -27,12 +27,11 @@ + diff --git a/test/perf/benchmarks/Benchmarks.cs b/test/perf/benchmarks/Benchmarks.cs deleted file mode 100644 index e0ff37018..000000000 --- a/test/perf/benchmarks/Benchmarks.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System; -using BenchmarkDotNet; -using BenchmarkDotNet.Attributes; - -namespace benchmarks -{ - public class Benchmarks - { - [Benchmark] - public void Scenario1() - { - //PSResourceInfo a = null; - - // Implement your benchmark here - - //return a; - } - - [Benchmark] - public void Scenario2() - { - // Implement your benchmark here - } - } -} diff --git a/test/perf/benchmarks/BenchmarksV2.cs b/test/perf/benchmarks/BenchmarksV2.cs new file mode 100644 index 000000000..952161c31 --- /dev/null +++ b/test/perf/benchmarks/BenchmarksV2.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using BenchmarkDotNet; +using BenchmarkDotNet.Attributes; +using System.Collections.ObjectModel; +using System.Management.Automation; +using Microsoft.PowerShell; + +namespace benchmarks +{ + public class BenchmarksV2 + { + System.Management.Automation.PowerShell pwsh; + + [IterationSetup] + public void IterationSetup() + { + var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); + //defaultSS.ImportPSModule(new string[] { }) + defaultSS.ExecutionPolicy = ExecutionPolicy.Unrestricted; + pwsh = System.Management.Automation.PowerShell.Create(defaultSS); + + // pwsh.AddScript("\"PShome var: $pshome\""); + pwsh.AddScript("Import-Module PowerShellGet -RequiredVersion 2.2.5 -Force"); + var results = pwsh.Invoke(); + } + + [IterationCleanup] + public void IterationCleanup() + { + // Disposing logic + pwsh.Dispose(); + } + + [Benchmark] + public void FindAzModuleV2() + { + Collection results = null; + + pwsh.AddScript("Find-Module -Name Az -Repository PSGallery"); + results = pwsh.Invoke(); + } + + [Benchmark] + public void FindAzModuleAndDependenciesV2() + { + Collection results = null; + + pwsh.AddScript("Find-Module -Name Az -IncludeDependencies -Repository PSGallery"); + results = pwsh.Invoke(); + } + + [Benchmark] + public void InstallAzModuleAndDependenciesV2() + { + Collection results = null; + + pwsh.AddScript("Install-Module -Name Az -Repository PSGallery -Force"); + results = pwsh.Invoke(); + } + } +} diff --git a/test/perf/benchmarks/BenchmarksV3.cs b/test/perf/benchmarks/BenchmarksV3.cs new file mode 100644 index 000000000..2eefe3c2d --- /dev/null +++ b/test/perf/benchmarks/BenchmarksV3.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using BenchmarkDotNet; +using BenchmarkDotNet.Attributes; +using System.Collections.ObjectModel; +using System.Management.Automation; +using Microsoft.PowerShell; + +namespace benchmarks +{ + public class BenchmarksV3 + { + System.Management.Automation.PowerShell pwsh; + + [IterationSetup] + public void IterationSetup() + { + var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); + defaultSS.ExecutionPolicy = ExecutionPolicy.Unrestricted; + pwsh = System.Management.Automation.PowerShell.Create(defaultSS); + + pwsh.AddScript("Import-Module PowerShellGet -RequiredVersion 3.0.14 -Force"); + var results = pwsh.Invoke(); + } + + [IterationCleanup] + public void IterationCleanup() + { + // Disposing logic + pwsh.Dispose(); + } + + [Benchmark] + public void FindAzModuleV3() + { + Collection results = null; + + pwsh.AddScript("Find-PSResource -Name Az -Repository PSGallery"); + results = pwsh.Invoke(); + } + + [Benchmark] + public void FindAzModuleAndDependenciesV3() + { + Collection results = null; + + pwsh.AddScript("Find-PSResource -Name Az -IncludeDependencies -Repository PSGallery"); + results = pwsh.Invoke(); + } + + [Benchmark] + public void InstallAzModuleV3() + { + Collection results = null; + + pwsh.AddScript("Install-PSResource -Name Az -Repository PSGallery -TrustRepository -SkipDependencyCheck -Reinstall"); + results = pwsh.Invoke(); + } + + [Benchmark] + public void InstallAzModuleAndDependenciesV3() + { + Collection results = null; + + pwsh.AddScript("Install-PSResource -Name Az -Repository PSGallery -TrustRepository -Reinstall"); + results = pwsh.Invoke(); + } + } +} diff --git a/test/perf/benchmarks/Program.cs b/test/perf/benchmarks/Program.cs index a73026086..83e534ab3 100644 --- a/test/perf/benchmarks/Program.cs +++ b/test/perf/benchmarks/Program.cs @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + using BenchmarkDotNet.Running; namespace benchmarks @@ -6,7 +9,8 @@ public sealed class Program { public static void Main(string[] args) { - var summary = BenchmarkRunner.Run(); + var summaryV2 = BenchmarkRunner.Run(); + var summaryV3 = BenchmarkRunner.Run(); } } } \ No newline at end of file diff --git a/test/perf/benchmarks/benchmarks.csproj b/test/perf/benchmarks/benchmarks.csproj index 1dc8da747..df4cdb248 100644 --- a/test/perf/benchmarks/benchmarks.csproj +++ b/test/perf/benchmarks/benchmarks.csproj @@ -20,17 +20,6 @@ - - - - - - - - - - - - + \ No newline at end of file From d7714b115aa6946557786479478515c6c3231265 Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 6 Jul 2022 13:54:44 -0700 Subject: [PATCH 06/14] Add tests for local repositories --- test/perf/benchmarks/BenchmarksV2LocalRepo.cs | 64 +++++++++++++++++ ...chmarksV2.cs => BenchmarksV2RemoteRepo.cs} | 25 +++---- test/perf/benchmarks/BenchmarksV3LocalRepo.cs | 70 +++++++++++++++++++ ...chmarksV3.cs => BenchmarksV3RemoteRepo.cs} | 29 +++----- test/perf/benchmarks/Program.cs | 6 +- 5 files changed, 158 insertions(+), 36 deletions(-) create mode 100644 test/perf/benchmarks/BenchmarksV2LocalRepo.cs rename test/perf/benchmarks/{BenchmarksV2.cs => BenchmarksV2RemoteRepo.cs} (72%) create mode 100644 test/perf/benchmarks/BenchmarksV3LocalRepo.cs rename test/perf/benchmarks/{BenchmarksV3.cs => BenchmarksV3RemoteRepo.cs} (76%) diff --git a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs new file mode 100644 index 000000000..5cbb50f63 --- /dev/null +++ b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using BenchmarkDotNet; +using BenchmarkDotNet.Attributes; +using Microsoft.PowerShell; +using System; +using System.Collections.ObjectModel; +using System.Management.Automation; + +namespace benchmarks +{ + public class BenchmarksV2LocalRepo + { + System.Management.Automation.PowerShell pwsh; + + [GlobalSetup] + public void GlobalSetup() + { + // Setting up the PowerShell runspace + var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); + defaultSS.ExecutionPolicy = ExecutionPolicy.Unrestricted; + pwsh = System.Management.Automation.PowerShell.Create(defaultSS); + + // Using PSGet v3 in order to save the Az modules and its dependencies + pwsh.AddScript("Import-Module PowerShellGet -RequiredVersion 3.0.14 -Force"); + pwsh.AddScript("New-Item TestRepo -ItemType Directory"); + pwsh.AddScript("Save-PSResource -Name Az -Repository PSGallery -AsNupkg -TrustRepository -Path .\\TestRepo"); + + // Now import the PSGet module version we want to test and register a local repo + pwsh.AddScript("Import-Module PowerShellGet -RequiredVersion 2.2.5 -Force"); + pwsh.AddScript("Register-PSRepository -Name LocalRepo -SourceLocation .\\TestRepo"); + + pwsh.Invoke(); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + pwsh.Dispose(); + } + + [Benchmark] + public void FindAzModuleV3() + { + pwsh.AddScript("Find-Module -Name Az -Repository LocalRepo"); + pwsh.Invoke(); + } + + [Benchmark] + public void FindAzModuleAndDependenciesV3() + { + pwsh.AddScript("Find-Module -Name Az -IncludeDependencies -Repository LocalRepo"); + pwsh.Invoke(); + } + + [Benchmark] + public void InstallAzModuleAndDependenciesV3() + { + pwsh.AddScript("Install-Module -Name Az -Repository LocalRepo -Force"); + pwsh.Invoke(); + } + } +} diff --git a/test/perf/benchmarks/BenchmarksV2.cs b/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs similarity index 72% rename from test/perf/benchmarks/BenchmarksV2.cs rename to test/perf/benchmarks/BenchmarksV2RemoteRepo.cs index 952161c31..484ac67c3 100644 --- a/test/perf/benchmarks/BenchmarksV2.cs +++ b/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs @@ -1,64 +1,57 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; using BenchmarkDotNet; using BenchmarkDotNet.Attributes; +using Microsoft.PowerShell; +using System; using System.Collections.ObjectModel; using System.Management.Automation; -using Microsoft.PowerShell; namespace benchmarks { - public class BenchmarksV2 + public class BenchmarksV2RemoteRepo { System.Management.Automation.PowerShell pwsh; [IterationSetup] public void IterationSetup() { + // Setting up the PowerShell runspace var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); - //defaultSS.ImportPSModule(new string[] { }) defaultSS.ExecutionPolicy = ExecutionPolicy.Unrestricted; pwsh = System.Management.Automation.PowerShell.Create(defaultSS); - // pwsh.AddScript("\"PShome var: $pshome\""); + // Import the PSGet version we want to test pwsh.AddScript("Import-Module PowerShellGet -RequiredVersion 2.2.5 -Force"); - var results = pwsh.Invoke(); + pwsh.Invoke(); } [IterationCleanup] public void IterationCleanup() { - // Disposing logic pwsh.Dispose(); } [Benchmark] public void FindAzModuleV2() { - Collection results = null; - pwsh.AddScript("Find-Module -Name Az -Repository PSGallery"); - results = pwsh.Invoke(); + pwsh.Invoke(); } [Benchmark] public void FindAzModuleAndDependenciesV2() { - Collection results = null; - pwsh.AddScript("Find-Module -Name Az -IncludeDependencies -Repository PSGallery"); - results = pwsh.Invoke(); + pwsh.Invoke(); } [Benchmark] public void InstallAzModuleAndDependenciesV2() { - Collection results = null; - pwsh.AddScript("Install-Module -Name Az -Repository PSGallery -Force"); - results = pwsh.Invoke(); + pwsh.Invoke(); } } } diff --git a/test/perf/benchmarks/BenchmarksV3LocalRepo.cs b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs new file mode 100644 index 000000000..ecdd3cf31 --- /dev/null +++ b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using BenchmarkDotNet; +using BenchmarkDotNet.Attributes; +using Microsoft.PowerShell; +using System; +using System.Collections.ObjectModel; +using System.Management.Automation; + +namespace benchmarks +{ + public class BenchmarksV3LocalRepo + { + System.Management.Automation.PowerShell pwsh; + + [GlobalSetup] + public void GlobalSetup() + { + // Setting up the PowerShell runspace + var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); + defaultSS.ExecutionPolicy = ExecutionPolicy.Unrestricted; + pwsh = System.Management.Automation.PowerShell.Create(defaultSS); + + // Import the PSGet module version we want to test, register a local repo, and save the Az modules and its dependencies + pwsh.AddScript("Import-Module PowerShellGet -RequiredVersion 3.0.14 -Force"); + pwsh.AddScript("New-Item TestRepo -ItemType Directory"); + pwsh.AddScript("Register-PSResourceRepository -Name LocalRepo -Uri .\\TestRepo"); + pwsh.AddScript("Save-PSResource -Name Az -Repository PSGallery -AsNupkg -TrustRepository -Path .\\TestRepo"); + + pwsh.Invoke(); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + pwsh.Dispose(); + } + + [Benchmark] + public void FindAzModuleV3() + { + pwsh.AddScript("Find-PSResource -Name Az -Repository LocalRepo"); + pwsh.Invoke(); + } + + [Benchmark] + public void FindAzModuleAndDependenciesV3() + { + Collection results = null; + + pwsh.AddScript("Find-PSResource -Name Az -IncludeDependencies -Repository LocalRepo"); + pwsh.Invoke(); + } + + [Benchmark] + public void InstallAzModuleV3() + { + pwsh.AddScript("Install-PSResource -Name Az -Repository LocalRepo -TrustRepository -SkipDependencyCheck -Reinstall"); + pwsh.Invoke(); + } + + [Benchmark] + public void InstallAzModuleAndDependenciesV3() + { + pwsh.AddScript("Install-PSResource -Name Az -Repository LocalRepo -TrustRepository -Reinstall"); + pwsh.Invoke(); + } + } +} diff --git a/test/perf/benchmarks/BenchmarksV3.cs b/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs similarity index 76% rename from test/perf/benchmarks/BenchmarksV3.cs rename to test/perf/benchmarks/BenchmarksV3RemoteRepo.cs index 2eefe3c2d..f5464ab03 100644 --- a/test/perf/benchmarks/BenchmarksV3.cs +++ b/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs @@ -1,71 +1,64 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; using BenchmarkDotNet; using BenchmarkDotNet.Attributes; +using Microsoft.PowerShell; +using System; using System.Collections.ObjectModel; using System.Management.Automation; -using Microsoft.PowerShell; namespace benchmarks { - public class BenchmarksV3 + public class BenchmarksV3RemoteRepo { System.Management.Automation.PowerShell pwsh; [IterationSetup] public void IterationSetup() { + // Setting up the PowerShell runspace var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); defaultSS.ExecutionPolicy = ExecutionPolicy.Unrestricted; pwsh = System.Management.Automation.PowerShell.Create(defaultSS); - + + // Import the PSGet version we want to test pwsh.AddScript("Import-Module PowerShellGet -RequiredVersion 3.0.14 -Force"); - var results = pwsh.Invoke(); + pwsh.Invoke(); } [IterationCleanup] public void IterationCleanup() { - // Disposing logic pwsh.Dispose(); } [Benchmark] public void FindAzModuleV3() { - Collection results = null; - pwsh.AddScript("Find-PSResource -Name Az -Repository PSGallery"); - results = pwsh.Invoke(); + pwsh.Invoke(); } [Benchmark] public void FindAzModuleAndDependenciesV3() { - Collection results = null; - pwsh.AddScript("Find-PSResource -Name Az -IncludeDependencies -Repository PSGallery"); - results = pwsh.Invoke(); + pwsh.Invoke(); } [Benchmark] public void InstallAzModuleV3() { - Collection results = null; - pwsh.AddScript("Install-PSResource -Name Az -Repository PSGallery -TrustRepository -SkipDependencyCheck -Reinstall"); - results = pwsh.Invoke(); + pwsh.Invoke(); } [Benchmark] public void InstallAzModuleAndDependenciesV3() { - Collection results = null; - pwsh.AddScript("Install-PSResource -Name Az -Repository PSGallery -TrustRepository -Reinstall"); - results = pwsh.Invoke(); + pwsh.Invoke(); } } } diff --git a/test/perf/benchmarks/Program.cs b/test/perf/benchmarks/Program.cs index 83e534ab3..1217c6285 100644 --- a/test/perf/benchmarks/Program.cs +++ b/test/perf/benchmarks/Program.cs @@ -9,8 +9,10 @@ public sealed class Program { public static void Main(string[] args) { - var summaryV2 = BenchmarkRunner.Run(); - var summaryV3 = BenchmarkRunner.Run(); + var summaryV2Remote = BenchmarkRunner.Run(); + var summaryV3Remote = BenchmarkRunner.Run(); + var summaryV2Local = BenchmarkRunner.Run(); + var summaryV3Local = BenchmarkRunner.Run(); } } } \ No newline at end of file From dd65b68277b2e8ab9ca3eb963d54e5b17cf1da5c Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 6 Jul 2022 13:59:50 -0700 Subject: [PATCH 07/14] Add readme --- test/perf/benchmarks/README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 test/perf/benchmarks/README.md diff --git a/test/perf/benchmarks/README.md b/test/perf/benchmarks/README.md new file mode 100644 index 000000000..127b164a7 --- /dev/null +++ b/test/perf/benchmarks/README.md @@ -0,0 +1,32 @@ +## Micro Benchmarks + +This folder contains micro benchmarks that test the performance of PowerShellGet. + +### Quick Start + +You can run the benchmarks directly using `dotnet run` in this directory: +1. To run the benchmarks in Interactive Mode, where you will be asked which benchmark(s) to run: + ``` + dotnet run -c Release -f net6.0 + ``` + +2. To list all available benchmarks ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Listing-the-Benchmarks)): + ``` + dotnet run -c Release -f net6.0 --list [flat/tree] + ``` + +3. To filter the benchmarks using a glob pattern applied to `namespace.typeName.methodName` ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Filtering-the-Benchmarks)]): + ``` + dotnet run -c Release -f net6.0 --filter *script* --list flat + ``` + +4. To profile the benchmarked code and produce an ETW Trace file ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Profiling)) + ``` + dotnet run -c Release -f net6.0 --filter *script* --profiler ETW + ``` + +## References + +- [Getting started with BenchmarkDotNet](https://benchmarkdotnet.org/articles/guides/getting-started.html) +- [Micro-benchmark Design Guidelines](https://github.com/dotnet/performance/blob/main/docs/microbenchmark-design-guidelines.md) +- [Adam SITNIK: Powerful benchmarking in .NET](https://www.youtube.com/watch?v=pdcrSG4tOLI&t=351s) \ No newline at end of file From 53b242c38beab31e657a89aca0b8ea8c813b27d8 Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 6 Jul 2022 14:02:33 -0700 Subject: [PATCH 08/14] Change 'benchmark' to 'Benchmark' --- src/code/AssemblyInfo.cs | 2 +- src/code/PowerShellGet.csproj | 6 ------ test/perf/benchmarks/BenchmarksV2LocalRepo.cs | 2 +- test/perf/benchmarks/BenchmarksV2RemoteRepo.cs | 2 +- test/perf/benchmarks/BenchmarksV3LocalRepo.cs | 2 +- test/perf/benchmarks/BenchmarksV3RemoteRepo.cs | 2 +- 6 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/code/AssemblyInfo.cs b/src/code/AssemblyInfo.cs index 95fcac8d6..1fa4d3538 100644 --- a/src/code/AssemblyInfo.cs +++ b/src/code/AssemblyInfo.cs @@ -3,4 +3,4 @@ using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("benchmarks")] \ No newline at end of file +[assembly: InternalsVisibleTo("Benchmarks")] \ No newline at end of file diff --git a/src/code/PowerShellGet.csproj b/src/code/PowerShellGet.csproj index c40f72d0f..dbd51ee95 100644 --- a/src/code/PowerShellGet.csproj +++ b/src/code/PowerShellGet.csproj @@ -27,12 +27,6 @@ - - diff --git a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs index 5cbb50f63..b81574a94 100644 --- a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs @@ -8,7 +8,7 @@ using System.Collections.ObjectModel; using System.Management.Automation; -namespace benchmarks +namespace Benchmarks { public class BenchmarksV2LocalRepo { diff --git a/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs b/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs index 484ac67c3..1cfa16321 100644 --- a/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs +++ b/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs @@ -8,7 +8,7 @@ using System.Collections.ObjectModel; using System.Management.Automation; -namespace benchmarks +namespace Benchmarks { public class BenchmarksV2RemoteRepo { diff --git a/test/perf/benchmarks/BenchmarksV3LocalRepo.cs b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs index ecdd3cf31..532f0ff87 100644 --- a/test/perf/benchmarks/BenchmarksV3LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs @@ -8,7 +8,7 @@ using System.Collections.ObjectModel; using System.Management.Automation; -namespace benchmarks +namespace Benchmarks { public class BenchmarksV3LocalRepo { diff --git a/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs b/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs index f5464ab03..eb399ddb1 100644 --- a/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs +++ b/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs @@ -8,7 +8,7 @@ using System.Collections.ObjectModel; using System.Management.Automation; -namespace benchmarks +namespace Benchmarks { public class BenchmarksV3RemoteRepo { From 93b0650cf923664449877e0a2cfe10b2cd096526 Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 6 Jul 2022 14:45:57 -0700 Subject: [PATCH 09/14] Minor fixes, move internals visible to csproj --- src/code/AssemblyInfo.cs | 6 ------ src/code/PowerShellGet.csproj | 4 ++++ test/perf/benchmarks/BenchmarksV3LocalRepo.cs | 2 -- test/perf/benchmarks/Program.cs | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 src/code/AssemblyInfo.cs diff --git a/src/code/AssemblyInfo.cs b/src/code/AssemblyInfo.cs deleted file mode 100644 index 1fa4d3538..000000000 --- a/src/code/AssemblyInfo.cs +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("Benchmarks")] \ No newline at end of file diff --git a/src/code/PowerShellGet.csproj b/src/code/PowerShellGet.csproj index dbd51ee95..e9d2e9a88 100644 --- a/src/code/PowerShellGet.csproj +++ b/src/code/PowerShellGet.csproj @@ -27,6 +27,10 @@ + + + + diff --git a/test/perf/benchmarks/BenchmarksV3LocalRepo.cs b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs index 532f0ff87..5eadd657b 100644 --- a/test/perf/benchmarks/BenchmarksV3LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs @@ -47,8 +47,6 @@ public void FindAzModuleV3() [Benchmark] public void FindAzModuleAndDependenciesV3() { - Collection results = null; - pwsh.AddScript("Find-PSResource -Name Az -IncludeDependencies -Repository LocalRepo"); pwsh.Invoke(); } diff --git a/test/perf/benchmarks/Program.cs b/test/perf/benchmarks/Program.cs index 1217c6285..5780a0ace 100644 --- a/test/perf/benchmarks/Program.cs +++ b/test/perf/benchmarks/Program.cs @@ -3,7 +3,7 @@ using BenchmarkDotNet.Running; -namespace benchmarks +namespace Benchmarks { public sealed class Program { From c2151a008608046584082bc02509aee5a2b6958f Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 6 Jul 2022 16:05:07 -0700 Subject: [PATCH 10/14] Clean up tests --- test/perf/benchmarks/BenchmarksV2LocalRepo.cs | 3 +++ test/perf/benchmarks/BenchmarksV2RemoteRepo.cs | 9 ++++++--- test/perf/benchmarks/BenchmarksV3LocalRepo.cs | 4 ++++ test/perf/benchmarks/BenchmarksV3RemoteRepo.cs | 12 ++++++++---- test/perf/benchmarks/benchmarks.csproj | 2 +- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs index b81574a94..65616bbad 100644 --- a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs @@ -43,6 +43,7 @@ public void GlobalCleanup() [Benchmark] public void FindAzModuleV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-Module -Name Az -Repository LocalRepo"); pwsh.Invoke(); } @@ -50,6 +51,7 @@ public void FindAzModuleV3() [Benchmark] public void FindAzModuleAndDependenciesV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-Module -Name Az -IncludeDependencies -Repository LocalRepo"); pwsh.Invoke(); } @@ -57,6 +59,7 @@ public void FindAzModuleAndDependenciesV3() [Benchmark] public void InstallAzModuleAndDependenciesV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Install-Module -Name Az -Repository LocalRepo -Force"); pwsh.Invoke(); } diff --git a/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs b/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs index 1cfa16321..cfa0cf88c 100644 --- a/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs +++ b/test/perf/benchmarks/BenchmarksV2RemoteRepo.cs @@ -14,8 +14,8 @@ public class BenchmarksV2RemoteRepo { System.Management.Automation.PowerShell pwsh; - [IterationSetup] - public void IterationSetup() + [GlobalSetup] + public void GlobalSetup() { // Setting up the PowerShell runspace var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); @@ -27,7 +27,7 @@ public void IterationSetup() pwsh.Invoke(); } - [IterationCleanup] + [GloblaCleanup] public void IterationCleanup() { pwsh.Dispose(); @@ -36,6 +36,7 @@ public void IterationCleanup() [Benchmark] public void FindAzModuleV2() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-Module -Name Az -Repository PSGallery"); pwsh.Invoke(); } @@ -43,6 +44,7 @@ public void FindAzModuleV2() [Benchmark] public void FindAzModuleAndDependenciesV2() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-Module -Name Az -IncludeDependencies -Repository PSGallery"); pwsh.Invoke(); } @@ -50,6 +52,7 @@ public void FindAzModuleAndDependenciesV2() [Benchmark] public void InstallAzModuleAndDependenciesV2() { + pwsh.Commands.Clear(); pwsh.AddScript("Install-Module -Name Az -Repository PSGallery -Force"); pwsh.Invoke(); } diff --git a/test/perf/benchmarks/BenchmarksV3LocalRepo.cs b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs index 5eadd657b..f3cfe6044 100644 --- a/test/perf/benchmarks/BenchmarksV3LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV3LocalRepo.cs @@ -40,6 +40,7 @@ public void GlobalCleanup() [Benchmark] public void FindAzModuleV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-PSResource -Name Az -Repository LocalRepo"); pwsh.Invoke(); } @@ -47,6 +48,7 @@ public void FindAzModuleV3() [Benchmark] public void FindAzModuleAndDependenciesV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-PSResource -Name Az -IncludeDependencies -Repository LocalRepo"); pwsh.Invoke(); } @@ -54,6 +56,7 @@ public void FindAzModuleAndDependenciesV3() [Benchmark] public void InstallAzModuleV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Install-PSResource -Name Az -Repository LocalRepo -TrustRepository -SkipDependencyCheck -Reinstall"); pwsh.Invoke(); } @@ -61,6 +64,7 @@ public void InstallAzModuleV3() [Benchmark] public void InstallAzModuleAndDependenciesV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Install-PSResource -Name Az -Repository LocalRepo -TrustRepository -Reinstall"); pwsh.Invoke(); } diff --git a/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs b/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs index eb399ddb1..278dd7535 100644 --- a/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs +++ b/test/perf/benchmarks/BenchmarksV3RemoteRepo.cs @@ -14,8 +14,8 @@ public class BenchmarksV3RemoteRepo { System.Management.Automation.PowerShell pwsh; - [IterationSetup] - public void IterationSetup() + [GlobalSetup] + public void GlobalSetup() { // Setting up the PowerShell runspace var defaultSS = System.Management.Automation.Runspaces.InitialSessionState.CreateDefault2(); @@ -27,8 +27,8 @@ public void IterationSetup() pwsh.Invoke(); } - [IterationCleanup] - public void IterationCleanup() + [GlobalCleanup] + public void GlobalCleanup() { pwsh.Dispose(); } @@ -36,6 +36,7 @@ public void IterationCleanup() [Benchmark] public void FindAzModuleV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-PSResource -Name Az -Repository PSGallery"); pwsh.Invoke(); } @@ -43,6 +44,7 @@ public void FindAzModuleV3() [Benchmark] public void FindAzModuleAndDependenciesV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Find-PSResource -Name Az -IncludeDependencies -Repository PSGallery"); pwsh.Invoke(); } @@ -50,6 +52,7 @@ public void FindAzModuleAndDependenciesV3() [Benchmark] public void InstallAzModuleV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Install-PSResource -Name Az -Repository PSGallery -TrustRepository -SkipDependencyCheck -Reinstall"); pwsh.Invoke(); } @@ -57,6 +60,7 @@ public void InstallAzModuleV3() [Benchmark] public void InstallAzModuleAndDependenciesV3() { + pwsh.Commands.Clear(); pwsh.AddScript("Install-PSResource -Name Az -Repository PSGallery -TrustRepository -Reinstall"); pwsh.Invoke(); } diff --git a/test/perf/benchmarks/benchmarks.csproj b/test/perf/benchmarks/benchmarks.csproj index df4cdb248..3ea7e4fe8 100644 --- a/test/perf/benchmarks/benchmarks.csproj +++ b/test/perf/benchmarks/benchmarks.csproj @@ -22,4 +22,4 @@ - \ No newline at end of file + From 800f8ba1bbab18e4a7e88fdb8520d2eba6a727d6 Mon Sep 17 00:00:00 2001 From: Amber Erickson Date: Wed, 6 Jul 2022 16:06:20 -0700 Subject: [PATCH 11/14] Add new line to program.cs file --- test/perf/benchmarks/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/benchmarks/Program.cs b/test/perf/benchmarks/Program.cs index 5780a0ace..a6e6ac316 100644 --- a/test/perf/benchmarks/Program.cs +++ b/test/perf/benchmarks/Program.cs @@ -15,4 +15,4 @@ public static void Main(string[] args) var summaryV3Local = BenchmarkRunner.Run(); } } -} \ No newline at end of file +} From fe670780a4366c3191b3e864a448967728c8e869 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 7 Jul 2022 11:25:36 -0700 Subject: [PATCH 12/14] Update test/perf/benchmarks/BenchmarksV2LocalRepo.cs Co-authored-by: Anam Navied --- test/perf/benchmarks/BenchmarksV2LocalRepo.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs index 65616bbad..2d9481ba7 100644 --- a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs @@ -41,7 +41,8 @@ public void GlobalCleanup() } [Benchmark] - public void FindAzModuleV3() + public void FindAzModuleV2() + { pwsh.Commands.Clear(); pwsh.AddScript("Find-Module -Name Az -Repository LocalRepo"); From ac8a03d094019f4216c5573e621f6765b6e9eb39 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 7 Jul 2022 11:25:50 -0700 Subject: [PATCH 13/14] Update test/perf/benchmarks/BenchmarksV2LocalRepo.cs Co-authored-by: Anam Navied --- test/perf/benchmarks/BenchmarksV2LocalRepo.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs index 2d9481ba7..d78e79098 100644 --- a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs @@ -58,7 +58,8 @@ public void FindAzModuleAndDependenciesV3() } [Benchmark] - public void InstallAzModuleAndDependenciesV3() + public void InstallAzModuleAndDependenciesV2() + { pwsh.Commands.Clear(); pwsh.AddScript("Install-Module -Name Az -Repository LocalRepo -Force"); From ecad5d7e8444ea1bfa1dd5a426cbfa93be9c5539 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 7 Jul 2022 11:25:58 -0700 Subject: [PATCH 14/14] Update test/perf/benchmarks/BenchmarksV2LocalRepo.cs Co-authored-by: Anam Navied --- test/perf/benchmarks/BenchmarksV2LocalRepo.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs index d78e79098..966db9c5b 100644 --- a/test/perf/benchmarks/BenchmarksV2LocalRepo.cs +++ b/test/perf/benchmarks/BenchmarksV2LocalRepo.cs @@ -50,7 +50,8 @@ public void FindAzModuleV2() } [Benchmark] - public void FindAzModuleAndDependenciesV3() + public void FindAzModuleAndDependenciesV2() + { pwsh.Commands.Clear(); pwsh.AddScript("Find-Module -Name Az -IncludeDependencies -Repository LocalRepo");