From 2b2d49a5b65d577b516ff8c02faa71502595eb2d Mon Sep 17 00:00:00 2001 From: Charlie Poole Date: Fri, 12 Jul 2024 11:12:33 -0700 Subject: [PATCH] Update recipe to match project loader --- NUnitConsole.sln | 58 ++--- build.cake | 19 +- cake/package-test.cake | 34 --- cake/test-runners.cake | 117 --------- cake/tools.cake | 4 - .../nunit.console-runner.netcore.nuspec | 2 +- {cake => recipe}/banner.cake | 0 {cake => recipe}/build-settings.cake | 11 +- {cake => recipe}/builder.cake | 0 {cake => recipe}/chocolatey-package.cake | 6 +- {cake => recipe}/command-line-options.cake | 0 {cake => recipe}/constants.cake | 3 + {cake => recipe}/dotnet.cake | 18 +- {cake => recipe}/extending.cake | 0 {cake => recipe}/headers.cake | 4 +- {cake => recipe}/help-messages.cake | 0 {cake => recipe}/known-extensions.cake | 0 {cake => recipe}/nuget-package.cake | 6 +- {cake => recipe}/package-checks.cake | 0 {cake => recipe}/package-definition.cake | 109 ++++++--- {cake => recipe}/package-reference.cake | 0 recipe/package-test.cake | 76 ++++++ {cake => recipe}/publishing.cake | 0 {cake => recipe}/setup.cake | 0 {cake => recipe}/task-builders.cake | 0 {cake => recipe}/task-definitions.cake | 0 {cake => recipe}/test-reports.cake | 16 +- {cake => recipe}/test-results.cake | 0 recipe/test-runners.cake | 224 ++++++++++++++++++ recipe/tools.cake | 27 +++ {cake => recipe}/unit-testing.cake | 5 +- {cake => recipe}/utilities.cake | 0 {cake => recipe}/versioning.cake | 0 {cake => recipe}/zip-package.cake | 8 +- 34 files changed, 492 insertions(+), 255 deletions(-) delete mode 100644 cake/package-test.cake delete mode 100644 cake/test-runners.cake delete mode 100644 cake/tools.cake rename {cake => recipe}/banner.cake (100%) rename {cake => recipe}/build-settings.cake (97%) rename {cake => recipe}/builder.cake (100%) rename {cake => recipe}/chocolatey-package.cake (86%) rename {cake => recipe}/command-line-options.cake (100%) rename {cake => recipe}/constants.cake (97%) rename {cake => recipe}/dotnet.cake (76%) rename {cake => recipe}/extending.cake (100%) rename {cake => recipe}/headers.cake (96%) rename {cake => recipe}/help-messages.cake (100%) rename {cake => recipe}/known-extensions.cake (100%) rename {cake => recipe}/nuget-package.cake (88%) rename {cake => recipe}/package-checks.cake (100%) rename {cake => recipe}/package-definition.cake (66%) rename {cake => recipe}/package-reference.cake (100%) create mode 100644 recipe/package-test.cake rename {cake => recipe}/publishing.cake (100%) rename {cake => recipe}/setup.cake (100%) rename {cake => recipe}/task-builders.cake (100%) rename {cake => recipe}/task-definitions.cake (100%) rename {cake => recipe}/test-reports.cake (93%) rename {cake => recipe}/test-results.cake (100%) create mode 100644 recipe/test-runners.cake create mode 100644 recipe/tools.cake rename {cake => recipe}/unit-testing.cake (96%) rename {cake => recipe}/utilities.cake (100%) rename {cake => recipe}/versioning.cake (100%) rename {cake => recipe}/zip-package.cake (90%) diff --git a/NUnitConsole.sln b/NUnitConsole.sln index dd041f3e7..ad28c0a8e 100644 --- a/NUnitConsole.sln +++ b/NUnitConsole.sln @@ -136,36 +136,36 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{C5B712 .config\dotnet-tools.json = .config\dotnet-tools.json EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cake", "cake", "{D6449B7A-20FF-467B-A65E-174DD6992AEB}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "recipe", "recipe", "{D6449B7A-20FF-467B-A65E-174DD6992AEB}" ProjectSection(SolutionItems) = preProject - cake\banner.cake = cake\banner.cake - cake\build-settings.cake = cake\build-settings.cake - cake\builder.cake = cake\builder.cake - cake\chocolatey-package.cake = cake\chocolatey-package.cake - cake\command-line-options.cake = cake\command-line-options.cake - cake\constants.cake = cake\constants.cake - cake\dotnet.cake = cake\dotnet.cake - cake\extending.cake = cake\extending.cake - cake\headers.cake = cake\headers.cake - cake\help-messages.cake = cake\help-messages.cake - cake\known-extensions.cake = cake\known-extensions.cake - cake\nuget-package.cake = cake\nuget-package.cake - cake\package-checks.cake = cake\package-checks.cake - cake\package-definition.cake = cake\package-definition.cake - cake\package-reference.cake = cake\package-reference.cake - cake\package-test.cake = cake\package-test.cake - cake\publishing.cake = cake\publishing.cake - cake\setup.cake = cake\setup.cake - cake\task-builders.cake = cake\task-builders.cake - cake\task-definitions.cake = cake\task-definitions.cake - cake\test-reports.cake = cake\test-reports.cake - cake\test-results.cake = cake\test-results.cake - cake\test-runners.cake = cake\test-runners.cake - cake\tools.cake = cake\tools.cake - cake\unit-testing.cake = cake\unit-testing.cake - cake\utilities.cake = cake\utilities.cake - cake\versioning.cake = cake\versioning.cake - cake\zip-package.cake = cake\zip-package.cake + recipe\banner.cake = recipe\banner.cake + recipe\build-settings.cake = recipe\build-settings.cake + recipe\builder.cake = recipe\builder.cake + recipe\chocolatey-package.cake = recipe\chocolatey-package.cake + recipe\command-line-options.cake = recipe\command-line-options.cake + recipe\constants.cake = recipe\constants.cake + recipe\dotnet.cake = recipe\dotnet.cake + recipe\extending.cake = recipe\extending.cake + recipe\headers.cake = recipe\headers.cake + recipe\help-messages.cake = recipe\help-messages.cake + recipe\known-extensions.cake = recipe\known-extensions.cake + recipe\nuget-package.cake = recipe\nuget-package.cake + recipe\package-checks.cake = recipe\package-checks.cake + recipe\package-definition.cake = recipe\package-definition.cake + recipe\package-reference.cake = recipe\package-reference.cake + recipe\package-test.cake = recipe\package-test.cake + recipe\publishing.cake = recipe\publishing.cake + recipe\setup.cake = recipe\setup.cake + recipe\task-builders.cake = recipe\task-builders.cake + recipe\task-definitions.cake = recipe\task-definitions.cake + recipe\test-reports.cake = recipe\test-reports.cake + recipe\test-results.cake = recipe\test-results.cake + recipe\test-runners.cake = recipe\test-runners.cake + recipe\tools.cake = recipe\tools.cake + recipe\unit-testing.cake = recipe\unit-testing.cake + recipe\utilities.cake = recipe\utilities.cake + recipe\versioning.cake = recipe\versioning.cake + recipe\zip-package.cake = recipe\zip-package.cake EndProjectSection EndProject Global diff --git a/build.cake b/build.cake index 824b230ce..6b55254f1 100644 --- a/build.cake +++ b/build.cake @@ -1,5 +1,5 @@ // Load scripts -#load cake/*.cake +#load recipe/*.cake // Initialize BuildSettings BuildSettings.Initialize( @@ -357,9 +357,9 @@ BuildSettings.Packages.AddRange(new PackageDefinition[] { // Custom unit test runner to run console vs engine tests differently // TODO: Use NUnitLite for all tests? -public class CustomTestRunner : UnitTestRunner +public class CustomTestRunner : TestRunner, IUnitTestRunner { - public override int Run(FilePath testPath) + public int RunUnitTest(FilePath testPath) { // Run console tests under the just-built console if(testPath.ToString().Contains("nunit3-console.tests.dll")) @@ -370,21 +370,24 @@ public class CustomTestRunner : UnitTestRunner } // All other tests use NUnitLite - return new NUnitLiteRunner().Run(testPath); + return new NUnitLiteRunner().RunUnitTest(testPath); } } // Use the console runner we just built to run package tests -public class ConsoleRunnerSelfTester : PackageTestRunner +public class ConsoleRunnerSelfTester : TestRunner, IPackageTestRunner { + private string _executablePath; + public ConsoleRunnerSelfTester(string executablePath) { - ExecutablePath = executablePath; + _executablePath = executablePath; } - public override int Run(string arguments) + public int RunPackageTest(string arguments) { - return base.Run(arguments); + Console.WriteLine("Running package test"); + return base.RunTest(_executablePath, arguments); } } diff --git a/cake/package-test.cake b/cake/package-test.cake deleted file mode 100644 index e80d160af..000000000 --- a/cake/package-test.cake +++ /dev/null @@ -1,34 +0,0 @@ -// Representation of a single test to be run against a pre-built package. -// Each test has a Level, with the following values defined... -// 0 Do not run - used for temporarily disabling a test -// 1 Run for all CI tests - that is every time we test packages -// 2 Run only on PRs, dev builds and when publishing -// 3 Run only when publishing -public struct PackageTest -{ - public int Level; - public string Name; - public string Description; - public string Arguments; - public ExpectedResult ExpectedResult; - public ExtensionSpecifier[] ExtensionsNeeded; - - public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params ExtensionSpecifier[] extensionsNeeded) - { - if (name == null) - throw new ArgumentNullException(nameof(name)); - if (description == null) - throw new ArgumentNullException(nameof(description)); - if (arguments == null) - throw new ArgumentNullException(nameof(arguments)); - if (expectedResult == null) - throw new ArgumentNullException(nameof(expectedResult)); - - Level = level; - Name = name; - Description = description; - Arguments = arguments; - ExpectedResult = expectedResult; - ExtensionsNeeded = extensionsNeeded; - } -} diff --git a/cake/test-runners.cake b/cake/test-runners.cake deleted file mode 100644 index d876d9c39..000000000 --- a/cake/test-runners.cake +++ /dev/null @@ -1,117 +0,0 @@ -/// -/// The TestRunner class is the abstract base for all TestRunners used to run unit- -/// or package-tests. A TestRunner knows how to run a test assembly and provide a result. -/// -public abstract class TestRunner -{ - public virtual bool RequiresInstallation => false; - - protected FilePath ExecutablePath { get; set; } - - // Base install does nothing - public virtual void Install() { } -} - -public abstract class UnitTestRunner : TestRunner -{ - // Unit tests are run by providing the path to a test assembly. - public virtual int Run(FilePath testAssembly) - { - if (ExecutablePath == null) - throw new InvalidOperationException("Unable to run tests. Executable path has not been set."); - - var processSettings = new ProcessSettings() { - WorkingDirectory = BuildSettings.OutputDirectory, - // HACK: Equality indicates we are running under NUnitLite - Arguments = ExecutablePath == testAssembly - ? BuildSettings.UnitTestArguments - : $"{testAssembly} {BuildSettings.UnitTestArguments}" }; - - if (ExecutablePath.GetExtension() == ".dll") - return BuildSettings.Context.StartProcess("dotnet", processSettings); - else - return BuildSettings.Context.StartProcess(ExecutablePath,processSettings); - } -} - -public abstract class PackageTestRunner : TestRunner -{ - // Package Tests are run by providing the arguments, which - // will include one or more test assemblies. - public virtual int Run(string arguments=null) - { - if (ExecutablePath == null) - throw new InvalidOperationException("Unable to run tests. Executable path has not been set."); - - var processSettings = new ProcessSettings() { WorkingDirectory = BuildSettings.OutputDirectory }; - - if (ExecutablePath.GetExtension() == ".dll") - { - processSettings.Arguments = $"{ExecutablePath} {arguments}"; - return BuildSettings.Context.StartProcess("dotnet", processSettings); - } - else - { - processSettings.Arguments = arguments; - return BuildSettings.Context.StartProcess(ExecutablePath, processSettings); - } - } -} - -/// -/// The InstallableTestRunner class is the abstract base for TestRunners which -/// must be installed using a published package before they can be used. -/// -public abstract class InstallableTestRunner : TestRunner -{ - public override bool RequiresInstallation => true; - - public InstallableTestRunner(string packageId, string version) - { - if (packageId == null) - throw new ArgumentNullException(nameof(packageId)); - if (version == null) - throw new ArgumentNullException(nameof(version)); - - PackageId = packageId; - Version = version; - } - - public string PackageId { get; } - public string Version { get; } - - public abstract string InstallPath { get; } -} - -public class NUnitLiteRunner : UnitTestRunner -{ - public override int Run(FilePath testPath) - { - ExecutablePath = testPath; - return base.Run(testPath); - } -} - -/// -/// Class that knows how to run an agent directly. -/// -public class AgentRunner : PackageTestRunner -{ - private string _stdExecutable; - private string _x86Executable; - - public AgentRunner(string stdExecutable, string x86Executable = null) - { - _stdExecutable = stdExecutable; - _x86Executable = x86Executable; - } - - public override int Run(string arguments) - { - ExecutablePath = arguments.Contains("--x86") - ? _x86Executable - : _stdExecutable; - - return base.Run(arguments.Replace("--x86", string.Empty)); - } -} diff --git a/cake/tools.cake b/cake/tools.cake deleted file mode 100644 index 749e35817..000000000 --- a/cake/tools.cake +++ /dev/null @@ -1,4 +0,0 @@ -// Load all tools used by the recipe -#tool NuGet.CommandLine&version=6.9.1 -#tool dotnet:?package=GitVersion.Tool&version=5.12.0 -#tool dotnet:?package=GitReleaseManager.Tool&version=0.17.0 diff --git a/nuget/runners/nunit.console-runner.netcore.nuspec b/nuget/runners/nunit.console-runner.netcore.nuspec index 5d44bdb9e..fd36e66a7 100644 --- a/nuget/runners/nunit.console-runner.netcore.nuspec +++ b/nuget/runners/nunit.console-runner.netcore.nuspec @@ -29,7 +29,7 @@ - + diff --git a/cake/banner.cake b/recipe/banner.cake similarity index 100% rename from cake/banner.cake rename to recipe/banner.cake diff --git a/cake/build-settings.cake b/recipe/build-settings.cake similarity index 97% rename from cake/build-settings.cake rename to recipe/build-settings.cake index 789735859..c503c798e 100644 --- a/cake/build-settings.cake +++ b/recipe/build-settings.cake @@ -21,7 +21,7 @@ public static class BuildSettings Verbosity msbuildVerbosity = Verbosity.Minimal, string unitTests = null, // Defaults to "**/*.tests.dll|**/*.tests.exe" (case insensitive) - UnitTestRunner unitTestRunner = null, // If not set, NUnitLite is used + IUnitTestRunner unitTestRunner = null, // If not set, NUnitLite is used string unitTestArguments = null ) { @@ -85,11 +85,11 @@ public static class BuildSettings // the solution found in the root directory provided there is only one. private static string DeduceSolutionFile() { - if (System.IO.File.Exists(Title + ".sln")) + if (SIO.File.Exists(Title + ".sln")) return Title + ".sln"; - var files = System.IO.Directory.GetFiles(ProjectDirectory, "*.sln"); - if (files.Length == 1 && System.IO.File.Exists(files[0])) + var files = SIO.Directory.GetFiles(ProjectDirectory, "*.sln"); + if (files.Length == 1 && SIO.File.Exists(files[0])) return files[0]; return null; @@ -203,7 +203,7 @@ public static class BuildSettings //Testing public static string UnitTests { get; set; } - public static UnitTestRunner UnitTestRunner { get; private set; } + public static IUnitTestRunner UnitTestRunner { get; private set; } public static string UnitTestArguments { get; private set; } // Packaging @@ -310,6 +310,7 @@ public static class BuildSettings Console.WriteLine("UnitTestRunner: " + UnitTestRunner?.GetType().Name ?? ""); Console.WriteLine("\nPACKAGING"); + Console.WriteLine("PackageTestLevel: " + PackageTestLevel); Console.WriteLine("MyGetPushUrl: " + MyGetPushUrl); Console.WriteLine("NuGetPushUrl: " + NuGetPushUrl); Console.WriteLine("ChocolateyPushUrl: " + ChocolateyPushUrl); diff --git a/cake/builder.cake b/recipe/builder.cake similarity index 100% rename from cake/builder.cake rename to recipe/builder.cake diff --git a/cake/chocolatey-package.cake b/recipe/chocolatey-package.cake similarity index 86% rename from cake/chocolatey-package.cake rename to recipe/chocolatey-package.cake index fa65eebed..7645d1bc8 100644 --- a/cake/chocolatey-package.cake +++ b/recipe/chocolatey-package.cake @@ -3,7 +3,8 @@ public class ChocolateyPackage : PackageDefinition public ChocolateyPackage( string id, string source, - PackageTestRunner testRunner = null, + IPackageTestRunner testRunner = null, + TestRunnerSource testRunnerSource = null, PackageCheck[] checks = null, IEnumerable tests = null) : base( @@ -11,6 +12,7 @@ public class ChocolateyPackage : PackageDefinition id, source, testRunner: testRunner, + testRunnerSource: testRunnerSource, checks: checks, tests: tests) { @@ -21,7 +23,7 @@ public class ChocolateyPackage : PackageDefinition // The file name of this package, including extension public override string PackageFileName => $"{PackageId}.{PackageVersion}.nupkg"; // The file name of any symbol package, including extension - public override string SymbolPackageName => System.IO.Path.ChangeExtension(PackageFileName, ".snupkg"); + public override string SymbolPackageName => SIO.Path.ChangeExtension(PackageFileName, ".snupkg"); // The directory into which this package is installed public override string PackageInstallDirectory => BuildSettings.ChocolateyTestDirectory; // The directory used to contain results of package tests for this package diff --git a/cake/command-line-options.cake b/recipe/command-line-options.cake similarity index 100% rename from cake/command-line-options.cake rename to recipe/command-line-options.cake diff --git a/cake/constants.cake b/recipe/constants.cake similarity index 97% rename from cake/constants.cake rename to recipe/constants.cake index 810238fb4..2a98b6873 100644 --- a/cake/constants.cake +++ b/recipe/constants.cake @@ -1,6 +1,9 @@ // This file contains both real constants and static readonly variables used // as constants. All values are initialized before any instance variables. +// Alias used throughout the recipe +using SIO = System.IO; + // GitHub owner is the NUnit organization const string GITHUB_OWNER = "nunit"; diff --git a/cake/dotnet.cake b/recipe/dotnet.cake similarity index 76% rename from cake/dotnet.cake rename to recipe/dotnet.cake index 0fb801824..9441de3ee 100644 --- a/cake/dotnet.cake +++ b/recipe/dotnet.cake @@ -18,7 +18,7 @@ public class DotnetInfo // NOTES: // * We don't need an IsInstalled property because our scripts all run under dotnet. - public bool IsX86Installed => System.IO.Directory.Exists(X86InstallPath) && System.IO.File.Exists(X86Executable); + public bool IsX86Installed => SIO.Directory.Exists(X86InstallPath) && SIO.File.Exists(X86Executable); public string InstallPath { get; } public string Executable => InstallPath + "dotnet.exe"; @@ -33,12 +33,12 @@ public class DotnetInfo _context.Information($"Install Path: {InstallPath}"); _context.Information($"Executable: {Executable}"); _context.Information("Runtimes:"); - foreach (string dir in System.IO.Directory.GetDirectories(System.IO.Path.Combine(InstallPath, "shared"))) + foreach (string dir in SIO.Directory.GetDirectories(SIO.Path.Combine(InstallPath, "shared"))) { - string runtime = System.IO.Path.GetFileName(dir); - foreach (string dir2 in System.IO.Directory.GetDirectories(dir)) + string runtime = SIO.Path.GetFileName(dir); + foreach (string dir2 in SIO.Directory.GetDirectories(dir)) { - string version = System.IO.Path.GetFileName(dir2); + string version = SIO.Path.GetFileName(dir2); _context.Information($" {runtime} {version}"); } } @@ -48,12 +48,12 @@ public class DotnetInfo _context.Information($"\nX86 Install Path: {X86InstallPath}"); _context.Information($"X86 Executable: {X86Executable}"); _context.Information("Runtimes:"); - foreach (var dir in System.IO.Directory.GetDirectories(System.IO.Path.Combine(X86InstallPath, "shared"))) + foreach (var dir in SIO.Directory.GetDirectories(SIO.Path.Combine(X86InstallPath, "shared"))) { - string runtime = System.IO.Path.GetFileName(dir); - foreach (string dir2 in System.IO.Directory.GetDirectories(dir)) + string runtime = SIO.Path.GetFileName(dir); + foreach (string dir2 in SIO.Directory.GetDirectories(dir)) { - string version = System.IO.Path.GetFileName(dir2); + string version = SIO.Path.GetFileName(dir2); _context.Information($" {runtime} {version}"); } } diff --git a/cake/extending.cake b/recipe/extending.cake similarity index 100% rename from cake/extending.cake rename to recipe/extending.cake diff --git a/cake/headers.cake b/recipe/headers.cake similarity index 96% rename from cake/headers.cake rename to recipe/headers.cake index 3ad3e77d4..479771469 100644 --- a/cake/headers.cake +++ b/recipe/headers.cake @@ -33,7 +33,7 @@ public static class Headers continue; // Ignore AssemblyInfo files - if (System.IO.Path.GetFileName(path) == "AssemblyInfo.cs") + if (SIO.Path.GetFileName(path) == "AssemblyInfo.cs") continue; examined++; @@ -92,7 +92,7 @@ public static class Headers List GetHeader(FilePath file) { var header = new List(); - var lines = System.IO.File.ReadLines(file.ToString()); + var lines = SIO.File.ReadLines(file.ToString()); foreach (string line in lines) { diff --git a/cake/help-messages.cake b/recipe/help-messages.cake similarity index 100% rename from cake/help-messages.cake rename to recipe/help-messages.cake diff --git a/cake/known-extensions.cake b/recipe/known-extensions.cake similarity index 100% rename from cake/known-extensions.cake rename to recipe/known-extensions.cake diff --git a/cake/nuget-package.cake b/recipe/nuget-package.cake similarity index 88% rename from cake/nuget-package.cake rename to recipe/nuget-package.cake index 979fb7c68..b784bf6aa 100644 --- a/cake/nuget-package.cake +++ b/recipe/nuget-package.cake @@ -3,7 +3,8 @@ public class NuGetPackage : PackageDefinition public NuGetPackage( string id, string source, - PackageTestRunner testRunner = null, + IPackageTestRunner testRunner = null, + TestRunnerSource testRunnerSource = null, PackageCheck[] checks = null, PackageCheck[] symbols = null, IEnumerable tests = null) @@ -12,6 +13,7 @@ public class NuGetPackage : PackageDefinition id, source, testRunner: testRunner, + testRunnerSource: testRunnerSource, checks: checks, symbols: symbols, tests: tests) @@ -29,7 +31,7 @@ public class NuGetPackage : PackageDefinition // The file name of this package, including extension public override string PackageFileName => $"{PackageId}.{PackageVersion}.nupkg"; // The file name of any symbol package, including extension - public override string SymbolPackageName => System.IO.Path.ChangeExtension(PackageFileName, ".snupkg"); + public override string SymbolPackageName => SIO.Path.ChangeExtension(PackageFileName, ".snupkg"); // The directory into which this package is installed public override string PackageInstallDirectory => BuildSettings.NuGetTestDirectory; // The directory used to contain results of package tests for this package diff --git a/cake/package-checks.cake b/recipe/package-checks.cake similarity index 100% rename from cake/package-checks.cake rename to recipe/package-checks.cake diff --git a/cake/package-definition.cake b/recipe/package-definition.cake similarity index 66% rename from cake/package-definition.cake rename to recipe/package-definition.cake index f97cb9fec..ebbc663b7 100644 --- a/cake/package-definition.cake +++ b/recipe/package-definition.cake @@ -23,14 +23,17 @@ public abstract class PackageDefinition PackageType packageType, string id, string source, - PackageTestRunner testRunner = null, + IPackageTestRunner testRunner = null, + TestRunnerSource testRunnerSource = null, string extraTestArguments = null, PackageCheck[] checks = null, PackageCheck[] symbols = null, IEnumerable tests = null) { - if (testRunner == null && tests != null) - throw new System.ArgumentException($"Unable to create {packageType} package {id}: TestRunner must be provided if there are tests", nameof(testRunner)); + if (testRunner == null && testRunnerSource == null && tests != null) + throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: TestRunner or TestRunnerSource must be provided if there are tests."); + if (testRunner != null && testRunnerSource != null) + throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: Either TestRunner or TestRunnerSource must be provided, but not both."); _context = BuildSettings.Context; @@ -40,6 +43,7 @@ public abstract class PackageDefinition PackageSource = source; BasePath = BuildSettings.OutputDirectory; TestRunner = testRunner; + TestRunnerSource = testRunnerSource; ExtraTestArguments = extraTestArguments; PackageChecks = checks; SymbolChecks = symbols; @@ -51,7 +55,8 @@ public abstract class PackageDefinition public string PackageVersion { get; protected set; } public string PackageSource { get; } public string BasePath { get; } - public PackageTestRunner TestRunner { get; } + public IPackageTestRunner TestRunner { get; } + public TestRunnerSource TestRunnerSource { get; } public string ExtraTestArguments { get; } public PackageCheck[] PackageChecks { get; } public PackageCheck[] SymbolChecks { get; protected set; } @@ -201,44 +206,55 @@ public abstract class PackageDefinition // _context.Information("Deleted directory " + dirPath.GetDirectoryName()); //} - //if (TestRunner.RequiresInstallation) - // TestRunner.Install(); + // Package was defined with either a TestRunnerSource or a single TestRunner. In either + // case, these will all be package test runners and may or may not require installation. + var defaultRunners = TestRunnerSource ?? new TestRunnerSource((TestRunner)TestRunner); + + // Preinstall all runners requiring installation + InstallRunners(defaultRunners.PackageTestRunners); foreach (var packageTest in PackageTests) { if (packageTest.Level > BuildSettings.PackageTestLevel) continue; - foreach (ExtensionSpecifier extension in packageTest.ExtensionsNeeded) - extension.InstallExtension(this); + InstallExtensions(packageTest.ExtensionsNeeded); + InstallRunners(packageTest.TestRunners); - var testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/"; - var resultFile = testResultDir + "TestResult.xml"; + // Use runners from the test if provided, otherwise the default runners + var runners = packageTest.TestRunners.Length > 0 ? packageTest.TestRunners : defaultRunners.PackageTestRunners; + + foreach (var runner in runners) + { + Console.WriteLine(runner.Version); + var testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/"; + var resultFile = testResultDir + "TestResult.xml"; - Banner.Display(packageTest.Description); + Banner.Display(packageTest.Description); - _context.CreateDirectory(testResultDir); - string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}"; - if (CommandLineOptions.TraceLevel.Value != "Off") - arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}"; + _context.CreateDirectory(testResultDir); + string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}"; + if (CommandLineOptions.TraceLevel.Value != "Off") + arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}"; - int rc = TestRunner.Run(arguments); + int rc = runner.RunPackageTest(arguments); - try - { - var result = new ActualResult(resultFile); - var report = new PackageTestReport(packageTest, result); - reporter.AddReport(report); + try + { + var result = new ActualResult(resultFile); + var report = new PackageTestReport(packageTest, result, runner); + reporter.AddReport(report); - Console.WriteLine(report.Errors.Count == 0 - ? "\nSUCCESS: Test Result matches expected result!" - : "\nERROR: Test Result not as expected!"); - } - catch (Exception ex) - { - reporter.AddReport(new PackageTestReport(packageTest, ex)); + Console.WriteLine(report.Errors.Count == 0 + ? "\nSUCCESS: Test Result matches expected result!" + : "\nERROR: Test Result not as expected!"); + } + catch (Exception ex) + { + reporter.AddReport(new PackageTestReport(packageTest, ex)); - Console.WriteLine("\nERROR: No result found!"); + Console.WriteLine("\nERROR: No result found!"); + } } } @@ -257,6 +273,41 @@ public abstract class PackageDefinition if (hadErrors) throw new Exception("One or more package tests had errors!"); } + + private void InstallExtensions(ExtensionSpecifier[] extensionsNeeded) + { + foreach (ExtensionSpecifier extension in extensionsNeeded) + extension.InstallExtension(this); + } + + private void InstallRunners(IEnumerable runners) + { + // Install any runners needing installation + foreach (var runner in runners) + if (runner is InstallableTestRunner) + InstallRunner((InstallableTestRunner)runner); + } + + private void InstallRunner(InstallableTestRunner runner) + { + runner.Install(PackageInstallDirectory); + + // We are using nuget packages for the runner, so it won't normally recognize + // chocolatey extensions. We add an extra addins file for that purpose. + if (PackageType == PackageType.Chocolatey) + { + var filePath = runner.ExecutablePath.GetDirectory().CombineWithFilePath("choco.engine.addins").ToString(); + Console.WriteLine($"Creating {filePath}"); + + using (var writer = new StreamWriter(filePath)) + { + writer.WriteLine("../../nunit-extension-*/tools/"); + writer.WriteLine("../../nunit-extension-*/tools/*/"); + writer.WriteLine("../../../nunit-extension-*/tools/"); + writer.WriteLine("../../../nunit-extension-*/tools/*/"); + } + } + } public virtual void VerifySymbolPackage() { } // Does nothing. Overridden for NuGet packages. } diff --git a/cake/package-reference.cake b/recipe/package-reference.cake similarity index 100% rename from cake/package-reference.cake rename to recipe/package-reference.cake diff --git a/recipe/package-test.cake b/recipe/package-test.cake new file mode 100644 index 000000000..98c291792 --- /dev/null +++ b/recipe/package-test.cake @@ -0,0 +1,76 @@ +// Representation of a single test to be run against a pre-built package. +// Each test has a Level, with the following values defined... +// 0 Do not run - used for temporarily disabling a test +// 1 Run for all CI tests - that is every time we test packages +// 2 Run only on PRs, dev builds and when publishing +// 3 Run only when publishing +public struct PackageTest +{ + public int Level; + public string Name; + public string Description; + public string Arguments; + public ExpectedResult ExpectedResult; + public IPackageTestRunner[] TestRunners; + public ExtensionSpecifier[] ExtensionsNeeded; + + public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult ) + { + if (name == null) + throw new ArgumentNullException(nameof(name)); + if (description == null) + throw new ArgumentNullException(nameof(description)); + if (arguments == null) + throw new ArgumentNullException(nameof(arguments)); + if (expectedResult == null) + throw new ArgumentNullException(nameof(expectedResult)); + + Level = level; + Name = name; + Description = description; + Arguments = arguments; + ExpectedResult = expectedResult; + ExtensionsNeeded = new ExtensionSpecifier[0]; + TestRunners = new IPackageTestRunner[0]; + } + + public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params ExtensionSpecifier[] extensionsNeeded ) + { + if (name == null) + throw new ArgumentNullException(nameof(name)); + if (description == null) + throw new ArgumentNullException(nameof(description)); + if (arguments == null) + throw new ArgumentNullException(nameof(arguments)); + if (expectedResult == null) + throw new ArgumentNullException(nameof(expectedResult)); + + Level = level; + Name = name; + Description = description; + Arguments = arguments; + ExpectedResult = expectedResult; + ExtensionsNeeded = extensionsNeeded; + TestRunners = new IPackageTestRunner[0]; + } + + public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params IPackageTestRunner[] testRunners ) + { + if (name == null) + throw new ArgumentNullException(nameof(name)); + if (description == null) + throw new ArgumentNullException(nameof(description)); + if (arguments == null) + throw new ArgumentNullException(nameof(arguments)); + if (expectedResult == null) + throw new ArgumentNullException(nameof(expectedResult)); + + Level = level; + Name = name; + Description = description; + Arguments = arguments; + ExpectedResult = expectedResult; + TestRunners = testRunners; + ExtensionsNeeded = new ExtensionSpecifier[0]; + } +} diff --git a/cake/publishing.cake b/recipe/publishing.cake similarity index 100% rename from cake/publishing.cake rename to recipe/publishing.cake diff --git a/cake/setup.cake b/recipe/setup.cake similarity index 100% rename from cake/setup.cake rename to recipe/setup.cake diff --git a/cake/task-builders.cake b/recipe/task-builders.cake similarity index 100% rename from cake/task-builders.cake rename to recipe/task-builders.cake diff --git a/cake/task-definitions.cake b/recipe/task-definitions.cake similarity index 100% rename from cake/task-definitions.cake rename to recipe/task-definitions.cake diff --git a/cake/test-reports.cake b/recipe/test-reports.cake similarity index 93% rename from cake/test-reports.cake rename to recipe/test-reports.cake index 9faf03d3d..3ecc9863f 100644 --- a/cake/test-reports.cake +++ b/recipe/test-reports.cake @@ -2,15 +2,15 @@ public class PackageTestReport { public PackageTest Test; public ActualResult Result; - public string ConsoleVersion; + public ITestRunner Runner; public List Errors; public List Warnings; - public PackageTestReport(PackageTest test, ActualResult actualResult, string consoleVersion = null) + public PackageTestReport(PackageTest test, ActualResult actualResult, ITestRunner runner = null) { Test = test; Result = actualResult; - ConsoleVersion = consoleVersion; + Runner = runner; Errors = new List(); Warnings = new List(); @@ -39,7 +39,7 @@ public class PackageTestReport if (expected.AssemblyName != actual.AssemblyName) Errors.Add($" Expected: {expected.AssemblyName} But was: { actual.AssemblyName}"); - else if (consoleVersion == null || !consoleVersion.StartsWith("NetCore.")) + else if (runner == null || runner.PackageId == "NUnit.ConsoleRunner.NetCore") { if (actual.Runtime == null) Warnings.Add($"Unable to determine actual runtime used for {expected.AssemblyName}"); @@ -55,21 +55,21 @@ public class PackageTestReport Errors.Add($" Found unexpected assembly {actualAssemblies[i].AssemblyName}"); } - public PackageTestReport(PackageTest test, Exception ex, string consoleVersion = null) + public PackageTestReport(PackageTest test, Exception ex, ITestRunner runner = null) { Test = test; Result = null; Errors = new List(); Errors.Add($" {ex.Message}"); - ConsoleVersion = consoleVersion; + Runner = runner; } public void Display(int index, TextWriter writer) { writer.WriteLine(); writer.WriteLine($"{index}. {Test.Description}"); - if (ConsoleVersion != null) - writer.WriteLine($" ConsoleVersion: {ConsoleVersion}"); + if (Runner != null) + writer.WriteLine($" Runner: {Runner.PackageId} {Runner.Version}"); writer.WriteLine($" Args: {Test.Arguments}"); writer.WriteLine(); diff --git a/cake/test-results.cake b/recipe/test-results.cake similarity index 100% rename from cake/test-results.cake rename to recipe/test-results.cake diff --git a/recipe/test-runners.cake b/recipe/test-runners.cake new file mode 100644 index 000000000..80f524dbe --- /dev/null +++ b/recipe/test-runners.cake @@ -0,0 +1,224 @@ +///////////////////////////////////////////////////////////////////////////// +// TEST RUNNER INTERFACES +///////////////////////////////////////////////////////////////////////////// + +/// +/// Common interface for all test runners +/// +public interface ITestRunner +{ + string PackageId { get; } + string Version { get; } +} + +/// +/// A runner capable of running unit tests +/// +public interface IUnitTestRunner : ITestRunner +{ + int RunUnitTest(FilePath testPath); +} + +/// +/// A runner capable of running package tests +/// +public interface IPackageTestRunner : ITestRunner +{ + int RunPackageTest(string arguments); +} + +///////////////////////////////////////////////////////////////////////////// +// ABSTRACT TEST RUNNER +///////////////////////////////////////////////////////////////////////////// + +/// +/// The TestRunner class is the abstract base for all TestRunners used to run unit- +/// or package-tests. A TestRunner knows how to run a test assembly and provide a result. +/// All base functionality is implemented in this class. Derived classes make that +/// functionality available selectively by implementing specific interfaces. +/// +public abstract class TestRunner : ITestRunner +{ + protected ICakeContext Context => BuildSettings.Context; + + public string PackageId { get; protected set; } + public string Version { get; protected set; } + + protected int RunTest(FilePath executablePath, string arguments = null) + { + return RunTest(executablePath, new ProcessSettings { Arguments = arguments }); + } + + protected int RunTest(FilePath executablePath, ProcessSettings processSettings=null) + { + if (executablePath == null) + throw new ArgumentNullException(nameof(executablePath)); + + if (processSettings == null) + processSettings = new ProcessSettings(); + + // Add default values to settings if not present + if (processSettings.WorkingDirectory == null) + processSettings.WorkingDirectory = BuildSettings.OutputDirectory; + + if (executablePath.GetExtension() == ".dll") + return Context.StartProcess("dotnet", processSettings); + else + return Context.StartProcess(executablePath, processSettings); + } +} + +/// +/// A TestRunner requiring some sort of installation before use. +/// +public abstract class InstallableTestRunner : TestRunner +{ + protected InstallableTestRunner(string packageId, string version) + { + PackageId = packageId; + Version = version; + } + + protected abstract FilePath ExecutableRelativePath { get; } + + // Path under tools directory where package would be installed by Cake #tool directive. + // NOTE: When used to run unit tests, a #tool directive is required. If derived package + // is only used for package tests, it is optional. + protected DirectoryPath ToolInstallDirectory => BuildSettings.ToolsDirectory + $"{PackageId}.{Version}"; + protected bool IsInstalledAsTool => + ToolInstallDirectory != null && Context.DirectoryExists(ToolInstallDirectory); + + protected DirectoryPath InstallDirectory; + + public FilePath ExecutablePath => InstallDirectory.CombineWithFilePath(ExecutableRelativePath); + + public void Install(DirectoryPath installDirectory) + { + InstallDirectory = installDirectory.Combine($"{PackageId}.{Version}"); + + // If the runner package is already installed as a cake tool, we just copy it + if (IsInstalledAsTool) + Context.CopyDirectory(ToolInstallDirectory, InstallDirectory); + // Otherwise, we install it to the requested location + else + Context.NuGetInstall( + PackageId, + new NuGetInstallSettings() { OutputDirectory = installDirectory, Version = Version }); + } +} + +///////////////////////////////////////////////////////////////////////////// +// TEST RUNNER SOURCE +///////////////////////////////////////////////////////////////////////////// + +/// +/// TestRunnerSource is a provider of TestRunners. It is used when the tests +/// are to be run under multiple TestRunners rather than just one. +/// +public class TestRunnerSource +{ + public TestRunnerSource(TestRunner runner1, params TestRunner[] moreRunners) + { + AllRunners.Add(runner1); + AllRunners.AddRange(moreRunners); + } + + public List AllRunners { get; } = new List(); + + public IEnumerable UnitTestRunners + { + get { foreach(var runner in AllRunners.Where(r => r is IUnitTestRunner)) yield return (IUnitTestRunner)runner; } + } + + public IEnumerable PackageTestRunners + { + get { foreach(var runner in AllRunners.Where(r => r is IPackageTestRunner)) yield return (IPackageTestRunner)runner; } + } +} + +///////////////////////////////////////////////////////////////////////////// +// NUNITLITE RUNNER +///////////////////////////////////////////////////////////////////////////// + +// For NUnitLite tests, the test is run directly +public class NUnitLiteRunner : TestRunner, IUnitTestRunner +{ + public int RunUnitTest(FilePath testPath) => + RunTest(testPath, BuildSettings.UnitTestArguments); +} + +///////////////////////////////////////////////////////////////////////////// +// NUNIT CONSOLE RUNNERS +///////////////////////////////////////////////////////////////////////////// + +// NUnitConsoleRunner is used for both unit and package tests. It must be pre-installed +// in the tools directory by use of a #tools directive. +public class NUnitConsoleRunner : InstallableTestRunner, IUnitTestRunner, IPackageTestRunner +{ + protected override FilePath ExecutableRelativePath => "tools/nunit3-console.exe"; + + public NUnitConsoleRunner(string version) : base("NUnit.ConsoleRunner", version) { } + + // Run a unit test + public int RunUnitTest(FilePath testPath) => RunTest(ToolInstallDirectory.CombineWithFilePath(ExecutableRelativePath), $"\"{testPath}\" {BuildSettings.UnitTestArguments}"); + + // Run a package test + public int RunPackageTest(string arguments) => RunTest(ExecutablePath, arguments); +} + +public class NUnitNetCoreConsoleRunner : InstallableTestRunner, IUnitTestRunner, IPackageTestRunner +{ + protected override FilePath ExecutableRelativePath => "tools/net6.0/nunit3-console.exe"; + + public NUnitNetCoreConsoleRunner(string version) : base("NUnit.ConsoleRunner.NetCore", version) { } + + // Run a unit test + public int RunUnitTest(FilePath testPath) => RunTest(ExecutablePath, $"\"{testPath}\" {BuildSettings.UnitTestArguments}"); + + // Run a package test + public int RunPackageTest(string arguments) => RunTest(ExecutablePath, arguments); +} + +public class EngineExtensionTestRunner : TestRunner, IPackageTestRunner +{ + private IPackageTestRunner[] _runners = new IPackageTestRunner[] { + new NUnitConsoleRunner("3.17.0"), + new NUnitConsoleRunner("3.15.5") + }; + + public int RunPackageTest(string arguments) + { + + return _runners[0].RunPackageTest(arguments); + } +} + +///////////////////////////////////////////////////////////////////////////// +// AGENT RUNNER +///////////////////////////////////////////////////////////////////////////// + +/// +/// Class that knows how to run an agent directly. (For future use) +/// +public class AgentRunner : TestRunner, IPackageTestRunner +{ + private string _stdExecutable; + private string _x86Executable; + + private FilePath _executablePath; + + public AgentRunner(string stdExecutable, string x86Executable = null) + { + _stdExecutable = stdExecutable; + _x86Executable = x86Executable; + } + + public int RunPackageTest(string arguments) + { + _executablePath = arguments.Contains("--x86") + ? _x86Executable + : _stdExecutable; + + return base.RunTest(_executablePath, arguments.Replace("--x86", string.Empty)); + } +} diff --git a/recipe/tools.cake b/recipe/tools.cake new file mode 100644 index 000000000..5aabb7680 --- /dev/null +++ b/recipe/tools.cake @@ -0,0 +1,27 @@ +// Load all tools used by the recipe +#tool NuGet.CommandLine&version=6.9.1 +#tool dotnet:?package=GitVersion.Tool&version=5.12.0 +#tool dotnet:?package=GitReleaseManager.Tool&version=0.17.0 + +public static class Tools +{ + public static DirectoryPath FindInstalledTool(string packageId) + { + if (SIO.Directory.Exists(BuildSettings.ToolsDirectory + packageId)) + return BuildSettings.ToolsDirectory + packageId; + + foreach(var dir in BuildSettings.Context.GetDirectories(BuildSettings.ToolsDirectory + $"{packageId}.*")) + return dir; // Use first one found + + return null; + } + + public static DirectoryPath FindInstalledTool(string packageId, string version) + { + if (version == null) + throw new ArgumentNullException(nameof(version)); + + var toolPath = BuildSettings.ToolsDirectory + $"{packageId}.{version}"; + return BuildSettings.ToolsDirectory + $"{packageId}.{version}"; + } +} \ No newline at end of file diff --git a/cake/unit-testing.cake b/recipe/unit-testing.cake similarity index 96% rename from cake/unit-testing.cake rename to recipe/unit-testing.cake index 766e8036d..24094d050 100644 --- a/cake/unit-testing.cake +++ b/recipe/unit-testing.cake @@ -18,6 +18,8 @@ public static class UnitTesting _context.Information($"Located {unitTests.Count} unit test assemblies."); var errors = new List(); + var runner = BuildSettings.UnitTestRunner ?? new NUnitLiteRunner(); + foreach (var testPath in unitTests) { var testFile = testPath.GetFilename(); @@ -28,8 +30,7 @@ public static class UnitTesting ? $"Running {testFile} under {runtime}" : $"Running {testFile}"); - var runner = BuildSettings.UnitTestRunner ?? new NUnitLiteRunner(); - int rc = runner.Run(testPath); + int rc = runner.RunUnitTest(testPath); var name = runtime != null ? $"{testFile}({runtime})" diff --git a/cake/utilities.cake b/recipe/utilities.cake similarity index 100% rename from cake/utilities.cake rename to recipe/utilities.cake diff --git a/cake/versioning.cake b/recipe/versioning.cake similarity index 100% rename from cake/versioning.cake rename to recipe/versioning.cake diff --git a/cake/zip-package.cake b/recipe/zip-package.cake similarity index 90% rename from cake/zip-package.cake rename to recipe/zip-package.cake index b10946349..4f27d03be 100644 --- a/cake/zip-package.cake +++ b/recipe/zip-package.cake @@ -3,7 +3,8 @@ public class ZipPackage : PackageDefinition public ZipPackage( string id, string source, - PackageTestRunner testRunner = null, + IPackageTestRunner testRunner = null, + TestRunnerSource testRunnerSource = null, PackageCheck[] checks = null, IEnumerable tests = null, PackageReference[] bundledExtensions = null ) @@ -11,7 +12,8 @@ public class ZipPackage : PackageDefinition PackageType.Zip, id, source, - testRunner: testRunner, + testRunner: testRunner, + testRunnerSource: testRunnerSource, checks: checks, tests: tests) { @@ -63,7 +65,7 @@ public class ZipPackage : PackageDefinition var addinsDir = BuildSettings.ZipImageDirectory + "bin/net462/addins/"; _context.CreateDirectory(addinsDir); - foreach (var packageDir in System.IO.Directory.GetDirectories(BuildSettings.ExtensionsDirectory)) + foreach (var packageDir in SIO.Directory.GetDirectories(BuildSettings.ExtensionsDirectory)) { var files = _context.GetFiles(packageDir + "/tools/*").Concat(_context.GetFiles(packageDir + "/tools/net462/*")); _context.CopyFiles(files.Where(f => f.GetExtension() != ".addins"), addinsDir);