diff --git a/NUnitConsole.sln b/NUnitConsole.sln
index dd041f3e7..ad28c0a8e 100644
--- a/NUnitConsole.sln
+++ b/NUnitConsole.sln
@@ -136,36 +136,36 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{C5B712
.config\dotnet-tools.json = .config\dotnet-tools.json
EndProjectSection
EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cake", "cake", "{D6449B7A-20FF-467B-A65E-174DD6992AEB}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "recipe", "recipe", "{D6449B7A-20FF-467B-A65E-174DD6992AEB}"
ProjectSection(SolutionItems) = preProject
- cake\banner.cake = cake\banner.cake
- cake\build-settings.cake = cake\build-settings.cake
- cake\builder.cake = cake\builder.cake
- cake\chocolatey-package.cake = cake\chocolatey-package.cake
- cake\command-line-options.cake = cake\command-line-options.cake
- cake\constants.cake = cake\constants.cake
- cake\dotnet.cake = cake\dotnet.cake
- cake\extending.cake = cake\extending.cake
- cake\headers.cake = cake\headers.cake
- cake\help-messages.cake = cake\help-messages.cake
- cake\known-extensions.cake = cake\known-extensions.cake
- cake\nuget-package.cake = cake\nuget-package.cake
- cake\package-checks.cake = cake\package-checks.cake
- cake\package-definition.cake = cake\package-definition.cake
- cake\package-reference.cake = cake\package-reference.cake
- cake\package-test.cake = cake\package-test.cake
- cake\publishing.cake = cake\publishing.cake
- cake\setup.cake = cake\setup.cake
- cake\task-builders.cake = cake\task-builders.cake
- cake\task-definitions.cake = cake\task-definitions.cake
- cake\test-reports.cake = cake\test-reports.cake
- cake\test-results.cake = cake\test-results.cake
- cake\test-runners.cake = cake\test-runners.cake
- cake\tools.cake = cake\tools.cake
- cake\unit-testing.cake = cake\unit-testing.cake
- cake\utilities.cake = cake\utilities.cake
- cake\versioning.cake = cake\versioning.cake
- cake\zip-package.cake = cake\zip-package.cake
+ recipe\banner.cake = recipe\banner.cake
+ recipe\build-settings.cake = recipe\build-settings.cake
+ recipe\builder.cake = recipe\builder.cake
+ recipe\chocolatey-package.cake = recipe\chocolatey-package.cake
+ recipe\command-line-options.cake = recipe\command-line-options.cake
+ recipe\constants.cake = recipe\constants.cake
+ recipe\dotnet.cake = recipe\dotnet.cake
+ recipe\extending.cake = recipe\extending.cake
+ recipe\headers.cake = recipe\headers.cake
+ recipe\help-messages.cake = recipe\help-messages.cake
+ recipe\known-extensions.cake = recipe\known-extensions.cake
+ recipe\nuget-package.cake = recipe\nuget-package.cake
+ recipe\package-checks.cake = recipe\package-checks.cake
+ recipe\package-definition.cake = recipe\package-definition.cake
+ recipe\package-reference.cake = recipe\package-reference.cake
+ recipe\package-test.cake = recipe\package-test.cake
+ recipe\publishing.cake = recipe\publishing.cake
+ recipe\setup.cake = recipe\setup.cake
+ recipe\task-builders.cake = recipe\task-builders.cake
+ recipe\task-definitions.cake = recipe\task-definitions.cake
+ recipe\test-reports.cake = recipe\test-reports.cake
+ recipe\test-results.cake = recipe\test-results.cake
+ recipe\test-runners.cake = recipe\test-runners.cake
+ recipe\tools.cake = recipe\tools.cake
+ recipe\unit-testing.cake = recipe\unit-testing.cake
+ recipe\utilities.cake = recipe\utilities.cake
+ recipe\versioning.cake = recipe\versioning.cake
+ recipe\zip-package.cake = recipe\zip-package.cake
EndProjectSection
EndProject
Global
diff --git a/build.cake b/build.cake
index 824b230ce..6b55254f1 100644
--- a/build.cake
+++ b/build.cake
@@ -1,5 +1,5 @@
// Load scripts
-#load cake/*.cake
+#load recipe/*.cake
// Initialize BuildSettings
BuildSettings.Initialize(
@@ -357,9 +357,9 @@ BuildSettings.Packages.AddRange(new PackageDefinition[] {
// Custom unit test runner to run console vs engine tests differently
// TODO: Use NUnitLite for all tests?
-public class CustomTestRunner : UnitTestRunner
+public class CustomTestRunner : TestRunner, IUnitTestRunner
{
- public override int Run(FilePath testPath)
+ public int RunUnitTest(FilePath testPath)
{
// Run console tests under the just-built console
if(testPath.ToString().Contains("nunit3-console.tests.dll"))
@@ -370,21 +370,24 @@ public class CustomTestRunner : UnitTestRunner
}
// All other tests use NUnitLite
- return new NUnitLiteRunner().Run(testPath);
+ return new NUnitLiteRunner().RunUnitTest(testPath);
}
}
// Use the console runner we just built to run package tests
-public class ConsoleRunnerSelfTester : PackageTestRunner
+public class ConsoleRunnerSelfTester : TestRunner, IPackageTestRunner
{
+ private string _executablePath;
+
public ConsoleRunnerSelfTester(string executablePath)
{
- ExecutablePath = executablePath;
+ _executablePath = executablePath;
}
- public override int Run(string arguments)
+ public int RunPackageTest(string arguments)
{
- return base.Run(arguments);
+ Console.WriteLine("Running package test");
+ return base.RunTest(_executablePath, arguments);
}
}
diff --git a/cake/package-test.cake b/cake/package-test.cake
deleted file mode 100644
index e80d160af..000000000
--- a/cake/package-test.cake
+++ /dev/null
@@ -1,34 +0,0 @@
-// Representation of a single test to be run against a pre-built package.
-// Each test has a Level, with the following values defined...
-// 0 Do not run - used for temporarily disabling a test
-// 1 Run for all CI tests - that is every time we test packages
-// 2 Run only on PRs, dev builds and when publishing
-// 3 Run only when publishing
-public struct PackageTest
-{
- public int Level;
- public string Name;
- public string Description;
- public string Arguments;
- public ExpectedResult ExpectedResult;
- public ExtensionSpecifier[] ExtensionsNeeded;
-
- public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params ExtensionSpecifier[] extensionsNeeded)
- {
- if (name == null)
- throw new ArgumentNullException(nameof(name));
- if (description == null)
- throw new ArgumentNullException(nameof(description));
- if (arguments == null)
- throw new ArgumentNullException(nameof(arguments));
- if (expectedResult == null)
- throw new ArgumentNullException(nameof(expectedResult));
-
- Level = level;
- Name = name;
- Description = description;
- Arguments = arguments;
- ExpectedResult = expectedResult;
- ExtensionsNeeded = extensionsNeeded;
- }
-}
diff --git a/cake/test-runners.cake b/cake/test-runners.cake
deleted file mode 100644
index d876d9c39..000000000
--- a/cake/test-runners.cake
+++ /dev/null
@@ -1,117 +0,0 @@
-///
-/// The TestRunner class is the abstract base for all TestRunners used to run unit-
-/// or package-tests. A TestRunner knows how to run a test assembly and provide a result.
-///
-public abstract class TestRunner
-{
- public virtual bool RequiresInstallation => false;
-
- protected FilePath ExecutablePath { get; set; }
-
- // Base install does nothing
- public virtual void Install() { }
-}
-
-public abstract class UnitTestRunner : TestRunner
-{
- // Unit tests are run by providing the path to a test assembly.
- public virtual int Run(FilePath testAssembly)
- {
- if (ExecutablePath == null)
- throw new InvalidOperationException("Unable to run tests. Executable path has not been set.");
-
- var processSettings = new ProcessSettings() {
- WorkingDirectory = BuildSettings.OutputDirectory,
- // HACK: Equality indicates we are running under NUnitLite
- Arguments = ExecutablePath == testAssembly
- ? BuildSettings.UnitTestArguments
- : $"{testAssembly} {BuildSettings.UnitTestArguments}" };
-
- if (ExecutablePath.GetExtension() == ".dll")
- return BuildSettings.Context.StartProcess("dotnet", processSettings);
- else
- return BuildSettings.Context.StartProcess(ExecutablePath,processSettings);
- }
-}
-
-public abstract class PackageTestRunner : TestRunner
-{
- // Package Tests are run by providing the arguments, which
- // will include one or more test assemblies.
- public virtual int Run(string arguments=null)
- {
- if (ExecutablePath == null)
- throw new InvalidOperationException("Unable to run tests. Executable path has not been set.");
-
- var processSettings = new ProcessSettings() { WorkingDirectory = BuildSettings.OutputDirectory };
-
- if (ExecutablePath.GetExtension() == ".dll")
- {
- processSettings.Arguments = $"{ExecutablePath} {arguments}";
- return BuildSettings.Context.StartProcess("dotnet", processSettings);
- }
- else
- {
- processSettings.Arguments = arguments;
- return BuildSettings.Context.StartProcess(ExecutablePath, processSettings);
- }
- }
-}
-
-///
-/// The InstallableTestRunner class is the abstract base for TestRunners which
-/// must be installed using a published package before they can be used.
-///
-public abstract class InstallableTestRunner : TestRunner
-{
- public override bool RequiresInstallation => true;
-
- public InstallableTestRunner(string packageId, string version)
- {
- if (packageId == null)
- throw new ArgumentNullException(nameof(packageId));
- if (version == null)
- throw new ArgumentNullException(nameof(version));
-
- PackageId = packageId;
- Version = version;
- }
-
- public string PackageId { get; }
- public string Version { get; }
-
- public abstract string InstallPath { get; }
-}
-
-public class NUnitLiteRunner : UnitTestRunner
-{
- public override int Run(FilePath testPath)
- {
- ExecutablePath = testPath;
- return base.Run(testPath);
- }
-}
-
-///
-/// Class that knows how to run an agent directly.
-///
-public class AgentRunner : PackageTestRunner
-{
- private string _stdExecutable;
- private string _x86Executable;
-
- public AgentRunner(string stdExecutable, string x86Executable = null)
- {
- _stdExecutable = stdExecutable;
- _x86Executable = x86Executable;
- }
-
- public override int Run(string arguments)
- {
- ExecutablePath = arguments.Contains("--x86")
- ? _x86Executable
- : _stdExecutable;
-
- return base.Run(arguments.Replace("--x86", string.Empty));
- }
-}
diff --git a/cake/tools.cake b/cake/tools.cake
deleted file mode 100644
index 749e35817..000000000
--- a/cake/tools.cake
+++ /dev/null
@@ -1,4 +0,0 @@
-// Load all tools used by the recipe
-#tool NuGet.CommandLine&version=6.9.1
-#tool dotnet:?package=GitVersion.Tool&version=5.12.0
-#tool dotnet:?package=GitReleaseManager.Tool&version=0.17.0
diff --git a/nuget/runners/nunit.console-runner.netcore.nuspec b/nuget/runners/nunit.console-runner.netcore.nuspec
index 5d44bdb9e..fd36e66a7 100644
--- a/nuget/runners/nunit.console-runner.netcore.nuspec
+++ b/nuget/runners/nunit.console-runner.netcore.nuspec
@@ -29,7 +29,7 @@
-
+
diff --git a/cake/banner.cake b/recipe/banner.cake
similarity index 100%
rename from cake/banner.cake
rename to recipe/banner.cake
diff --git a/cake/build-settings.cake b/recipe/build-settings.cake
similarity index 97%
rename from cake/build-settings.cake
rename to recipe/build-settings.cake
index 789735859..c503c798e 100644
--- a/cake/build-settings.cake
+++ b/recipe/build-settings.cake
@@ -21,7 +21,7 @@ public static class BuildSettings
Verbosity msbuildVerbosity = Verbosity.Minimal,
string unitTests = null, // Defaults to "**/*.tests.dll|**/*.tests.exe" (case insensitive)
- UnitTestRunner unitTestRunner = null, // If not set, NUnitLite is used
+ IUnitTestRunner unitTestRunner = null, // If not set, NUnitLite is used
string unitTestArguments = null
)
{
@@ -85,11 +85,11 @@ public static class BuildSettings
// the solution found in the root directory provided there is only one.
private static string DeduceSolutionFile()
{
- if (System.IO.File.Exists(Title + ".sln"))
+ if (SIO.File.Exists(Title + ".sln"))
return Title + ".sln";
- var files = System.IO.Directory.GetFiles(ProjectDirectory, "*.sln");
- if (files.Length == 1 && System.IO.File.Exists(files[0]))
+ var files = SIO.Directory.GetFiles(ProjectDirectory, "*.sln");
+ if (files.Length == 1 && SIO.File.Exists(files[0]))
return files[0];
return null;
@@ -203,7 +203,7 @@ public static class BuildSettings
//Testing
public static string UnitTests { get; set; }
- public static UnitTestRunner UnitTestRunner { get; private set; }
+ public static IUnitTestRunner UnitTestRunner { get; private set; }
public static string UnitTestArguments { get; private set; }
// Packaging
@@ -310,6 +310,7 @@ public static class BuildSettings
Console.WriteLine("UnitTestRunner: " + UnitTestRunner?.GetType().Name ?? "");
Console.WriteLine("\nPACKAGING");
+ Console.WriteLine("PackageTestLevel: " + PackageTestLevel);
Console.WriteLine("MyGetPushUrl: " + MyGetPushUrl);
Console.WriteLine("NuGetPushUrl: " + NuGetPushUrl);
Console.WriteLine("ChocolateyPushUrl: " + ChocolateyPushUrl);
diff --git a/cake/builder.cake b/recipe/builder.cake
similarity index 100%
rename from cake/builder.cake
rename to recipe/builder.cake
diff --git a/cake/chocolatey-package.cake b/recipe/chocolatey-package.cake
similarity index 86%
rename from cake/chocolatey-package.cake
rename to recipe/chocolatey-package.cake
index fa65eebed..7645d1bc8 100644
--- a/cake/chocolatey-package.cake
+++ b/recipe/chocolatey-package.cake
@@ -3,7 +3,8 @@ public class ChocolateyPackage : PackageDefinition
public ChocolateyPackage(
string id,
string source,
- PackageTestRunner testRunner = null,
+ IPackageTestRunner testRunner = null,
+ TestRunnerSource testRunnerSource = null,
PackageCheck[] checks = null,
IEnumerable tests = null)
: base(
@@ -11,6 +12,7 @@ public class ChocolateyPackage : PackageDefinition
id,
source,
testRunner: testRunner,
+ testRunnerSource: testRunnerSource,
checks: checks,
tests: tests)
{
@@ -21,7 +23,7 @@ public class ChocolateyPackage : PackageDefinition
// The file name of this package, including extension
public override string PackageFileName => $"{PackageId}.{PackageVersion}.nupkg";
// The file name of any symbol package, including extension
- public override string SymbolPackageName => System.IO.Path.ChangeExtension(PackageFileName, ".snupkg");
+ public override string SymbolPackageName => SIO.Path.ChangeExtension(PackageFileName, ".snupkg");
// The directory into which this package is installed
public override string PackageInstallDirectory => BuildSettings.ChocolateyTestDirectory;
// The directory used to contain results of package tests for this package
diff --git a/cake/command-line-options.cake b/recipe/command-line-options.cake
similarity index 100%
rename from cake/command-line-options.cake
rename to recipe/command-line-options.cake
diff --git a/cake/constants.cake b/recipe/constants.cake
similarity index 97%
rename from cake/constants.cake
rename to recipe/constants.cake
index 810238fb4..2a98b6873 100644
--- a/cake/constants.cake
+++ b/recipe/constants.cake
@@ -1,6 +1,9 @@
// This file contains both real constants and static readonly variables used
// as constants. All values are initialized before any instance variables.
+// Alias used throughout the recipe
+using SIO = System.IO;
+
// GitHub owner is the NUnit organization
const string GITHUB_OWNER = "nunit";
diff --git a/cake/dotnet.cake b/recipe/dotnet.cake
similarity index 76%
rename from cake/dotnet.cake
rename to recipe/dotnet.cake
index 0fb801824..9441de3ee 100644
--- a/cake/dotnet.cake
+++ b/recipe/dotnet.cake
@@ -18,7 +18,7 @@ public class DotnetInfo
// NOTES:
// * We don't need an IsInstalled property because our scripts all run under dotnet.
- public bool IsX86Installed => System.IO.Directory.Exists(X86InstallPath) && System.IO.File.Exists(X86Executable);
+ public bool IsX86Installed => SIO.Directory.Exists(X86InstallPath) && SIO.File.Exists(X86Executable);
public string InstallPath { get; }
public string Executable => InstallPath + "dotnet.exe";
@@ -33,12 +33,12 @@ public class DotnetInfo
_context.Information($"Install Path: {InstallPath}");
_context.Information($"Executable: {Executable}");
_context.Information("Runtimes:");
- foreach (string dir in System.IO.Directory.GetDirectories(System.IO.Path.Combine(InstallPath, "shared")))
+ foreach (string dir in SIO.Directory.GetDirectories(SIO.Path.Combine(InstallPath, "shared")))
{
- string runtime = System.IO.Path.GetFileName(dir);
- foreach (string dir2 in System.IO.Directory.GetDirectories(dir))
+ string runtime = SIO.Path.GetFileName(dir);
+ foreach (string dir2 in SIO.Directory.GetDirectories(dir))
{
- string version = System.IO.Path.GetFileName(dir2);
+ string version = SIO.Path.GetFileName(dir2);
_context.Information($" {runtime} {version}");
}
}
@@ -48,12 +48,12 @@ public class DotnetInfo
_context.Information($"\nX86 Install Path: {X86InstallPath}");
_context.Information($"X86 Executable: {X86Executable}");
_context.Information("Runtimes:");
- foreach (var dir in System.IO.Directory.GetDirectories(System.IO.Path.Combine(X86InstallPath, "shared")))
+ foreach (var dir in SIO.Directory.GetDirectories(SIO.Path.Combine(X86InstallPath, "shared")))
{
- string runtime = System.IO.Path.GetFileName(dir);
- foreach (string dir2 in System.IO.Directory.GetDirectories(dir))
+ string runtime = SIO.Path.GetFileName(dir);
+ foreach (string dir2 in SIO.Directory.GetDirectories(dir))
{
- string version = System.IO.Path.GetFileName(dir2);
+ string version = SIO.Path.GetFileName(dir2);
_context.Information($" {runtime} {version}");
}
}
diff --git a/cake/extending.cake b/recipe/extending.cake
similarity index 100%
rename from cake/extending.cake
rename to recipe/extending.cake
diff --git a/cake/headers.cake b/recipe/headers.cake
similarity index 96%
rename from cake/headers.cake
rename to recipe/headers.cake
index 3ad3e77d4..479771469 100644
--- a/cake/headers.cake
+++ b/recipe/headers.cake
@@ -33,7 +33,7 @@ public static class Headers
continue;
// Ignore AssemblyInfo files
- if (System.IO.Path.GetFileName(path) == "AssemblyInfo.cs")
+ if (SIO.Path.GetFileName(path) == "AssemblyInfo.cs")
continue;
examined++;
@@ -92,7 +92,7 @@ public static class Headers
List GetHeader(FilePath file)
{
var header = new List();
- var lines = System.IO.File.ReadLines(file.ToString());
+ var lines = SIO.File.ReadLines(file.ToString());
foreach (string line in lines)
{
diff --git a/cake/help-messages.cake b/recipe/help-messages.cake
similarity index 100%
rename from cake/help-messages.cake
rename to recipe/help-messages.cake
diff --git a/cake/known-extensions.cake b/recipe/known-extensions.cake
similarity index 100%
rename from cake/known-extensions.cake
rename to recipe/known-extensions.cake
diff --git a/cake/nuget-package.cake b/recipe/nuget-package.cake
similarity index 88%
rename from cake/nuget-package.cake
rename to recipe/nuget-package.cake
index 979fb7c68..b784bf6aa 100644
--- a/cake/nuget-package.cake
+++ b/recipe/nuget-package.cake
@@ -3,7 +3,8 @@ public class NuGetPackage : PackageDefinition
public NuGetPackage(
string id,
string source,
- PackageTestRunner testRunner = null,
+ IPackageTestRunner testRunner = null,
+ TestRunnerSource testRunnerSource = null,
PackageCheck[] checks = null,
PackageCheck[] symbols = null,
IEnumerable tests = null)
@@ -12,6 +13,7 @@ public class NuGetPackage : PackageDefinition
id,
source,
testRunner: testRunner,
+ testRunnerSource: testRunnerSource,
checks: checks,
symbols: symbols,
tests: tests)
@@ -29,7 +31,7 @@ public class NuGetPackage : PackageDefinition
// The file name of this package, including extension
public override string PackageFileName => $"{PackageId}.{PackageVersion}.nupkg";
// The file name of any symbol package, including extension
- public override string SymbolPackageName => System.IO.Path.ChangeExtension(PackageFileName, ".snupkg");
+ public override string SymbolPackageName => SIO.Path.ChangeExtension(PackageFileName, ".snupkg");
// The directory into which this package is installed
public override string PackageInstallDirectory => BuildSettings.NuGetTestDirectory;
// The directory used to contain results of package tests for this package
diff --git a/cake/package-checks.cake b/recipe/package-checks.cake
similarity index 100%
rename from cake/package-checks.cake
rename to recipe/package-checks.cake
diff --git a/cake/package-definition.cake b/recipe/package-definition.cake
similarity index 66%
rename from cake/package-definition.cake
rename to recipe/package-definition.cake
index f97cb9fec..ebbc663b7 100644
--- a/cake/package-definition.cake
+++ b/recipe/package-definition.cake
@@ -23,14 +23,17 @@ public abstract class PackageDefinition
PackageType packageType,
string id,
string source,
- PackageTestRunner testRunner = null,
+ IPackageTestRunner testRunner = null,
+ TestRunnerSource testRunnerSource = null,
string extraTestArguments = null,
PackageCheck[] checks = null,
PackageCheck[] symbols = null,
IEnumerable tests = null)
{
- if (testRunner == null && tests != null)
- throw new System.ArgumentException($"Unable to create {packageType} package {id}: TestRunner must be provided if there are tests", nameof(testRunner));
+ if (testRunner == null && testRunnerSource == null && tests != null)
+ throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: TestRunner or TestRunnerSource must be provided if there are tests.");
+ if (testRunner != null && testRunnerSource != null)
+ throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: Either TestRunner or TestRunnerSource must be provided, but not both.");
_context = BuildSettings.Context;
@@ -40,6 +43,7 @@ public abstract class PackageDefinition
PackageSource = source;
BasePath = BuildSettings.OutputDirectory;
TestRunner = testRunner;
+ TestRunnerSource = testRunnerSource;
ExtraTestArguments = extraTestArguments;
PackageChecks = checks;
SymbolChecks = symbols;
@@ -51,7 +55,8 @@ public abstract class PackageDefinition
public string PackageVersion { get; protected set; }
public string PackageSource { get; }
public string BasePath { get; }
- public PackageTestRunner TestRunner { get; }
+ public IPackageTestRunner TestRunner { get; }
+ public TestRunnerSource TestRunnerSource { get; }
public string ExtraTestArguments { get; }
public PackageCheck[] PackageChecks { get; }
public PackageCheck[] SymbolChecks { get; protected set; }
@@ -201,44 +206,55 @@ public abstract class PackageDefinition
// _context.Information("Deleted directory " + dirPath.GetDirectoryName());
//}
- //if (TestRunner.RequiresInstallation)
- // TestRunner.Install();
+ // Package was defined with either a TestRunnerSource or a single TestRunner. In either
+ // case, these will all be package test runners and may or may not require installation.
+ var defaultRunners = TestRunnerSource ?? new TestRunnerSource((TestRunner)TestRunner);
+
+ // Preinstall all runners requiring installation
+ InstallRunners(defaultRunners.PackageTestRunners);
foreach (var packageTest in PackageTests)
{
if (packageTest.Level > BuildSettings.PackageTestLevel)
continue;
- foreach (ExtensionSpecifier extension in packageTest.ExtensionsNeeded)
- extension.InstallExtension(this);
+ InstallExtensions(packageTest.ExtensionsNeeded);
+ InstallRunners(packageTest.TestRunners);
- var testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/";
- var resultFile = testResultDir + "TestResult.xml";
+ // Use runners from the test if provided, otherwise the default runners
+ var runners = packageTest.TestRunners.Length > 0 ? packageTest.TestRunners : defaultRunners.PackageTestRunners;
+
+ foreach (var runner in runners)
+ {
+ Console.WriteLine(runner.Version);
+ var testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/";
+ var resultFile = testResultDir + "TestResult.xml";
- Banner.Display(packageTest.Description);
+ Banner.Display(packageTest.Description);
- _context.CreateDirectory(testResultDir);
- string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}";
- if (CommandLineOptions.TraceLevel.Value != "Off")
- arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}";
+ _context.CreateDirectory(testResultDir);
+ string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}";
+ if (CommandLineOptions.TraceLevel.Value != "Off")
+ arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}";
- int rc = TestRunner.Run(arguments);
+ int rc = runner.RunPackageTest(arguments);
- try
- {
- var result = new ActualResult(resultFile);
- var report = new PackageTestReport(packageTest, result);
- reporter.AddReport(report);
+ try
+ {
+ var result = new ActualResult(resultFile);
+ var report = new PackageTestReport(packageTest, result, runner);
+ reporter.AddReport(report);
- Console.WriteLine(report.Errors.Count == 0
- ? "\nSUCCESS: Test Result matches expected result!"
- : "\nERROR: Test Result not as expected!");
- }
- catch (Exception ex)
- {
- reporter.AddReport(new PackageTestReport(packageTest, ex));
+ Console.WriteLine(report.Errors.Count == 0
+ ? "\nSUCCESS: Test Result matches expected result!"
+ : "\nERROR: Test Result not as expected!");
+ }
+ catch (Exception ex)
+ {
+ reporter.AddReport(new PackageTestReport(packageTest, ex));
- Console.WriteLine("\nERROR: No result found!");
+ Console.WriteLine("\nERROR: No result found!");
+ }
}
}
@@ -257,6 +273,41 @@ public abstract class PackageDefinition
if (hadErrors)
throw new Exception("One or more package tests had errors!");
}
+
+ private void InstallExtensions(ExtensionSpecifier[] extensionsNeeded)
+ {
+ foreach (ExtensionSpecifier extension in extensionsNeeded)
+ extension.InstallExtension(this);
+ }
+
+ private void InstallRunners(IEnumerable runners)
+ {
+ // Install any runners needing installation
+ foreach (var runner in runners)
+ if (runner is InstallableTestRunner)
+ InstallRunner((InstallableTestRunner)runner);
+ }
+
+ private void InstallRunner(InstallableTestRunner runner)
+ {
+ runner.Install(PackageInstallDirectory);
+
+ // We are using nuget packages for the runner, so it won't normally recognize
+ // chocolatey extensions. We add an extra addins file for that purpose.
+ if (PackageType == PackageType.Chocolatey)
+ {
+ var filePath = runner.ExecutablePath.GetDirectory().CombineWithFilePath("choco.engine.addins").ToString();
+ Console.WriteLine($"Creating {filePath}");
+
+ using (var writer = new StreamWriter(filePath))
+ {
+ writer.WriteLine("../../nunit-extension-*/tools/");
+ writer.WriteLine("../../nunit-extension-*/tools/*/");
+ writer.WriteLine("../../../nunit-extension-*/tools/");
+ writer.WriteLine("../../../nunit-extension-*/tools/*/");
+ }
+ }
+ }
public virtual void VerifySymbolPackage() { } // Does nothing. Overridden for NuGet packages.
}
diff --git a/cake/package-reference.cake b/recipe/package-reference.cake
similarity index 100%
rename from cake/package-reference.cake
rename to recipe/package-reference.cake
diff --git a/recipe/package-test.cake b/recipe/package-test.cake
new file mode 100644
index 000000000..98c291792
--- /dev/null
+++ b/recipe/package-test.cake
@@ -0,0 +1,76 @@
+// Representation of a single test to be run against a pre-built package.
+// Each test has a Level, with the following values defined...
+// 0 Do not run - used for temporarily disabling a test
+// 1 Run for all CI tests - that is every time we test packages
+// 2 Run only on PRs, dev builds and when publishing
+// 3 Run only when publishing
+public struct PackageTest
+{
+ public int Level;
+ public string Name;
+ public string Description;
+ public string Arguments;
+ public ExpectedResult ExpectedResult;
+ public IPackageTestRunner[] TestRunners;
+ public ExtensionSpecifier[] ExtensionsNeeded;
+
+ public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult )
+ {
+ if (name == null)
+ throw new ArgumentNullException(nameof(name));
+ if (description == null)
+ throw new ArgumentNullException(nameof(description));
+ if (arguments == null)
+ throw new ArgumentNullException(nameof(arguments));
+ if (expectedResult == null)
+ throw new ArgumentNullException(nameof(expectedResult));
+
+ Level = level;
+ Name = name;
+ Description = description;
+ Arguments = arguments;
+ ExpectedResult = expectedResult;
+ ExtensionsNeeded = new ExtensionSpecifier[0];
+ TestRunners = new IPackageTestRunner[0];
+ }
+
+ public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params ExtensionSpecifier[] extensionsNeeded )
+ {
+ if (name == null)
+ throw new ArgumentNullException(nameof(name));
+ if (description == null)
+ throw new ArgumentNullException(nameof(description));
+ if (arguments == null)
+ throw new ArgumentNullException(nameof(arguments));
+ if (expectedResult == null)
+ throw new ArgumentNullException(nameof(expectedResult));
+
+ Level = level;
+ Name = name;
+ Description = description;
+ Arguments = arguments;
+ ExpectedResult = expectedResult;
+ ExtensionsNeeded = extensionsNeeded;
+ TestRunners = new IPackageTestRunner[0];
+ }
+
+ public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params IPackageTestRunner[] testRunners )
+ {
+ if (name == null)
+ throw new ArgumentNullException(nameof(name));
+ if (description == null)
+ throw new ArgumentNullException(nameof(description));
+ if (arguments == null)
+ throw new ArgumentNullException(nameof(arguments));
+ if (expectedResult == null)
+ throw new ArgumentNullException(nameof(expectedResult));
+
+ Level = level;
+ Name = name;
+ Description = description;
+ Arguments = arguments;
+ ExpectedResult = expectedResult;
+ TestRunners = testRunners;
+ ExtensionsNeeded = new ExtensionSpecifier[0];
+ }
+}
diff --git a/cake/publishing.cake b/recipe/publishing.cake
similarity index 100%
rename from cake/publishing.cake
rename to recipe/publishing.cake
diff --git a/cake/setup.cake b/recipe/setup.cake
similarity index 100%
rename from cake/setup.cake
rename to recipe/setup.cake
diff --git a/cake/task-builders.cake b/recipe/task-builders.cake
similarity index 100%
rename from cake/task-builders.cake
rename to recipe/task-builders.cake
diff --git a/cake/task-definitions.cake b/recipe/task-definitions.cake
similarity index 100%
rename from cake/task-definitions.cake
rename to recipe/task-definitions.cake
diff --git a/cake/test-reports.cake b/recipe/test-reports.cake
similarity index 93%
rename from cake/test-reports.cake
rename to recipe/test-reports.cake
index 9faf03d3d..3ecc9863f 100644
--- a/cake/test-reports.cake
+++ b/recipe/test-reports.cake
@@ -2,15 +2,15 @@ public class PackageTestReport
{
public PackageTest Test;
public ActualResult Result;
- public string ConsoleVersion;
+ public ITestRunner Runner;
public List Errors;
public List Warnings;
- public PackageTestReport(PackageTest test, ActualResult actualResult, string consoleVersion = null)
+ public PackageTestReport(PackageTest test, ActualResult actualResult, ITestRunner runner = null)
{
Test = test;
Result = actualResult;
- ConsoleVersion = consoleVersion;
+ Runner = runner;
Errors = new List();
Warnings = new List();
@@ -39,7 +39,7 @@ public class PackageTestReport
if (expected.AssemblyName != actual.AssemblyName)
Errors.Add($" Expected: {expected.AssemblyName} But was: { actual.AssemblyName}");
- else if (consoleVersion == null || !consoleVersion.StartsWith("NetCore."))
+ else if (runner == null || runner.PackageId == "NUnit.ConsoleRunner.NetCore")
{
if (actual.Runtime == null)
Warnings.Add($"Unable to determine actual runtime used for {expected.AssemblyName}");
@@ -55,21 +55,21 @@ public class PackageTestReport
Errors.Add($" Found unexpected assembly {actualAssemblies[i].AssemblyName}");
}
- public PackageTestReport(PackageTest test, Exception ex, string consoleVersion = null)
+ public PackageTestReport(PackageTest test, Exception ex, ITestRunner runner = null)
{
Test = test;
Result = null;
Errors = new List();
Errors.Add($" {ex.Message}");
- ConsoleVersion = consoleVersion;
+ Runner = runner;
}
public void Display(int index, TextWriter writer)
{
writer.WriteLine();
writer.WriteLine($"{index}. {Test.Description}");
- if (ConsoleVersion != null)
- writer.WriteLine($" ConsoleVersion: {ConsoleVersion}");
+ if (Runner != null)
+ writer.WriteLine($" Runner: {Runner.PackageId} {Runner.Version}");
writer.WriteLine($" Args: {Test.Arguments}");
writer.WriteLine();
diff --git a/cake/test-results.cake b/recipe/test-results.cake
similarity index 100%
rename from cake/test-results.cake
rename to recipe/test-results.cake
diff --git a/recipe/test-runners.cake b/recipe/test-runners.cake
new file mode 100644
index 000000000..80f524dbe
--- /dev/null
+++ b/recipe/test-runners.cake
@@ -0,0 +1,224 @@
+/////////////////////////////////////////////////////////////////////////////
+// TEST RUNNER INTERFACES
+/////////////////////////////////////////////////////////////////////////////
+
+///
+/// Common interface for all test runners
+///
+public interface ITestRunner
+{
+ string PackageId { get; }
+ string Version { get; }
+}
+
+///
+/// A runner capable of running unit tests
+///
+public interface IUnitTestRunner : ITestRunner
+{
+ int RunUnitTest(FilePath testPath);
+}
+
+///
+/// A runner capable of running package tests
+///
+public interface IPackageTestRunner : ITestRunner
+{
+ int RunPackageTest(string arguments);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// ABSTRACT TEST RUNNER
+/////////////////////////////////////////////////////////////////////////////
+
+///
+/// The TestRunner class is the abstract base for all TestRunners used to run unit-
+/// or package-tests. A TestRunner knows how to run a test assembly and provide a result.
+/// All base functionality is implemented in this class. Derived classes make that
+/// functionality available selectively by implementing specific interfaces.
+///
+public abstract class TestRunner : ITestRunner
+{
+ protected ICakeContext Context => BuildSettings.Context;
+
+ public string PackageId { get; protected set; }
+ public string Version { get; protected set; }
+
+ protected int RunTest(FilePath executablePath, string arguments = null)
+ {
+ return RunTest(executablePath, new ProcessSettings { Arguments = arguments });
+ }
+
+ protected int RunTest(FilePath executablePath, ProcessSettings processSettings=null)
+ {
+ if (executablePath == null)
+ throw new ArgumentNullException(nameof(executablePath));
+
+ if (processSettings == null)
+ processSettings = new ProcessSettings();
+
+ // Add default values to settings if not present
+ if (processSettings.WorkingDirectory == null)
+ processSettings.WorkingDirectory = BuildSettings.OutputDirectory;
+
+ if (executablePath.GetExtension() == ".dll")
+ return Context.StartProcess("dotnet", processSettings);
+ else
+ return Context.StartProcess(executablePath, processSettings);
+ }
+}
+
+///
+/// A TestRunner requiring some sort of installation before use.
+///
+public abstract class InstallableTestRunner : TestRunner
+{
+ protected InstallableTestRunner(string packageId, string version)
+ {
+ PackageId = packageId;
+ Version = version;
+ }
+
+ protected abstract FilePath ExecutableRelativePath { get; }
+
+ // Path under tools directory where package would be installed by Cake #tool directive.
+ // NOTE: When used to run unit tests, a #tool directive is required. If derived package
+ // is only used for package tests, it is optional.
+ protected DirectoryPath ToolInstallDirectory => BuildSettings.ToolsDirectory + $"{PackageId}.{Version}";
+ protected bool IsInstalledAsTool =>
+ ToolInstallDirectory != null && Context.DirectoryExists(ToolInstallDirectory);
+
+ protected DirectoryPath InstallDirectory;
+
+ public FilePath ExecutablePath => InstallDirectory.CombineWithFilePath(ExecutableRelativePath);
+
+ public void Install(DirectoryPath installDirectory)
+ {
+ InstallDirectory = installDirectory.Combine($"{PackageId}.{Version}");
+
+ // If the runner package is already installed as a cake tool, we just copy it
+ if (IsInstalledAsTool)
+ Context.CopyDirectory(ToolInstallDirectory, InstallDirectory);
+ // Otherwise, we install it to the requested location
+ else
+ Context.NuGetInstall(
+ PackageId,
+ new NuGetInstallSettings() { OutputDirectory = installDirectory, Version = Version });
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// TEST RUNNER SOURCE
+/////////////////////////////////////////////////////////////////////////////
+
+///
+/// TestRunnerSource is a provider of TestRunners. It is used when the tests
+/// are to be run under multiple TestRunners rather than just one.
+///
+public class TestRunnerSource
+{
+ public TestRunnerSource(TestRunner runner1, params TestRunner[] moreRunners)
+ {
+ AllRunners.Add(runner1);
+ AllRunners.AddRange(moreRunners);
+ }
+
+ public List AllRunners { get; } = new List();
+
+ public IEnumerable UnitTestRunners
+ {
+ get { foreach(var runner in AllRunners.Where(r => r is IUnitTestRunner)) yield return (IUnitTestRunner)runner; }
+ }
+
+ public IEnumerable PackageTestRunners
+ {
+ get { foreach(var runner in AllRunners.Where(r => r is IPackageTestRunner)) yield return (IPackageTestRunner)runner; }
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// NUNITLITE RUNNER
+/////////////////////////////////////////////////////////////////////////////
+
+// For NUnitLite tests, the test is run directly
+public class NUnitLiteRunner : TestRunner, IUnitTestRunner
+{
+ public int RunUnitTest(FilePath testPath) =>
+ RunTest(testPath, BuildSettings.UnitTestArguments);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// NUNIT CONSOLE RUNNERS
+/////////////////////////////////////////////////////////////////////////////
+
+// NUnitConsoleRunner is used for both unit and package tests. It must be pre-installed
+// in the tools directory by use of a #tools directive.
+public class NUnitConsoleRunner : InstallableTestRunner, IUnitTestRunner, IPackageTestRunner
+{
+ protected override FilePath ExecutableRelativePath => "tools/nunit3-console.exe";
+
+ public NUnitConsoleRunner(string version) : base("NUnit.ConsoleRunner", version) { }
+
+ // Run a unit test
+ public int RunUnitTest(FilePath testPath) => RunTest(ToolInstallDirectory.CombineWithFilePath(ExecutableRelativePath), $"\"{testPath}\" {BuildSettings.UnitTestArguments}");
+
+ // Run a package test
+ public int RunPackageTest(string arguments) => RunTest(ExecutablePath, arguments);
+}
+
+public class NUnitNetCoreConsoleRunner : InstallableTestRunner, IUnitTestRunner, IPackageTestRunner
+{
+ protected override FilePath ExecutableRelativePath => "tools/net6.0/nunit3-console.exe";
+
+ public NUnitNetCoreConsoleRunner(string version) : base("NUnit.ConsoleRunner.NetCore", version) { }
+
+ // Run a unit test
+ public int RunUnitTest(FilePath testPath) => RunTest(ExecutablePath, $"\"{testPath}\" {BuildSettings.UnitTestArguments}");
+
+ // Run a package test
+ public int RunPackageTest(string arguments) => RunTest(ExecutablePath, arguments);
+}
+
+public class EngineExtensionTestRunner : TestRunner, IPackageTestRunner
+{
+ private IPackageTestRunner[] _runners = new IPackageTestRunner[] {
+ new NUnitConsoleRunner("3.17.0"),
+ new NUnitConsoleRunner("3.15.5")
+ };
+
+ public int RunPackageTest(string arguments)
+ {
+
+ return _runners[0].RunPackageTest(arguments);
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// AGENT RUNNER
+/////////////////////////////////////////////////////////////////////////////
+
+///
+/// Class that knows how to run an agent directly. (For future use)
+///
+public class AgentRunner : TestRunner, IPackageTestRunner
+{
+ private string _stdExecutable;
+ private string _x86Executable;
+
+ private FilePath _executablePath;
+
+ public AgentRunner(string stdExecutable, string x86Executable = null)
+ {
+ _stdExecutable = stdExecutable;
+ _x86Executable = x86Executable;
+ }
+
+ public int RunPackageTest(string arguments)
+ {
+ _executablePath = arguments.Contains("--x86")
+ ? _x86Executable
+ : _stdExecutable;
+
+ return base.RunTest(_executablePath, arguments.Replace("--x86", string.Empty));
+ }
+}
diff --git a/recipe/tools.cake b/recipe/tools.cake
new file mode 100644
index 000000000..5aabb7680
--- /dev/null
+++ b/recipe/tools.cake
@@ -0,0 +1,27 @@
+// Load all tools used by the recipe
+#tool NuGet.CommandLine&version=6.9.1
+#tool dotnet:?package=GitVersion.Tool&version=5.12.0
+#tool dotnet:?package=GitReleaseManager.Tool&version=0.17.0
+
+public static class Tools
+{
+ public static DirectoryPath FindInstalledTool(string packageId)
+ {
+ if (SIO.Directory.Exists(BuildSettings.ToolsDirectory + packageId))
+ return BuildSettings.ToolsDirectory + packageId;
+
+ foreach(var dir in BuildSettings.Context.GetDirectories(BuildSettings.ToolsDirectory + $"{packageId}.*"))
+ return dir; // Use first one found
+
+ return null;
+ }
+
+ public static DirectoryPath FindInstalledTool(string packageId, string version)
+ {
+ if (version == null)
+ throw new ArgumentNullException(nameof(version));
+
+ var toolPath = BuildSettings.ToolsDirectory + $"{packageId}.{version}";
+ return BuildSettings.ToolsDirectory + $"{packageId}.{version}";
+ }
+}
\ No newline at end of file
diff --git a/cake/unit-testing.cake b/recipe/unit-testing.cake
similarity index 96%
rename from cake/unit-testing.cake
rename to recipe/unit-testing.cake
index 766e8036d..24094d050 100644
--- a/cake/unit-testing.cake
+++ b/recipe/unit-testing.cake
@@ -18,6 +18,8 @@ public static class UnitTesting
_context.Information($"Located {unitTests.Count} unit test assemblies.");
var errors = new List();
+ var runner = BuildSettings.UnitTestRunner ?? new NUnitLiteRunner();
+
foreach (var testPath in unitTests)
{
var testFile = testPath.GetFilename();
@@ -28,8 +30,7 @@ public static class UnitTesting
? $"Running {testFile} under {runtime}"
: $"Running {testFile}");
- var runner = BuildSettings.UnitTestRunner ?? new NUnitLiteRunner();
- int rc = runner.Run(testPath);
+ int rc = runner.RunUnitTest(testPath);
var name = runtime != null
? $"{testFile}({runtime})"
diff --git a/cake/utilities.cake b/recipe/utilities.cake
similarity index 100%
rename from cake/utilities.cake
rename to recipe/utilities.cake
diff --git a/cake/versioning.cake b/recipe/versioning.cake
similarity index 100%
rename from cake/versioning.cake
rename to recipe/versioning.cake
diff --git a/cake/zip-package.cake b/recipe/zip-package.cake
similarity index 90%
rename from cake/zip-package.cake
rename to recipe/zip-package.cake
index b10946349..4f27d03be 100644
--- a/cake/zip-package.cake
+++ b/recipe/zip-package.cake
@@ -3,7 +3,8 @@ public class ZipPackage : PackageDefinition
public ZipPackage(
string id,
string source,
- PackageTestRunner testRunner = null,
+ IPackageTestRunner testRunner = null,
+ TestRunnerSource testRunnerSource = null,
PackageCheck[] checks = null,
IEnumerable tests = null,
PackageReference[] bundledExtensions = null )
@@ -11,7 +12,8 @@ public class ZipPackage : PackageDefinition
PackageType.Zip,
id,
source,
- testRunner: testRunner,
+ testRunner: testRunner,
+ testRunnerSource: testRunnerSource,
checks: checks,
tests: tests)
{
@@ -63,7 +65,7 @@ public class ZipPackage : PackageDefinition
var addinsDir = BuildSettings.ZipImageDirectory + "bin/net462/addins/";
_context.CreateDirectory(addinsDir);
- foreach (var packageDir in System.IO.Directory.GetDirectories(BuildSettings.ExtensionsDirectory))
+ foreach (var packageDir in SIO.Directory.GetDirectories(BuildSettings.ExtensionsDirectory))
{
var files = _context.GetFiles(packageDir + "/tools/*").Concat(_context.GetFiles(packageDir + "/tools/net462/*"));
_context.CopyFiles(files.Where(f => f.GetExtension() != ".addins"), addinsDir);