diff --git a/src/app/FakeLib/FakeLib.fsproj b/src/app/FakeLib/FakeLib.fsproj index 0b58fefe9fd..d66dcf7bb2b 100644 --- a/src/app/FakeLib/FakeLib.fsproj +++ b/src/app/FakeLib/FakeLib.fsproj @@ -65,6 +65,7 @@ + diff --git a/src/app/FakeLib/UnitTest/XUnit2Helper.fs b/src/app/FakeLib/UnitTest/XUnit2Helper.fs new file mode 100644 index 00000000000..45481bbb1c9 --- /dev/null +++ b/src/app/FakeLib/UnitTest/XUnit2Helper.fs @@ -0,0 +1,191 @@ +/// Contains tasks to run [xUnit](http://xunit.codeplex.com/) unit tests. +module Fake.XUnit2Helper + +open System +open System.IO +open System.Text + +(* +xUnit.net console test runner (64-bit .NET 4.0.30319.34209) +Copyright (C) 2014 Outercurve Foundation. + +usage: xunit.console [configFile] [options] + +Valid options: + -parallel option : set parallelization based on option + : none - turn off all parallelization + : collections - only parallelize collections + : assemblies - only parallelize assemblies + : all - parallelize assemblies & collections + -maxthreads count : maximum thread count for collection parallelization + : 0 - run with unbounded thread count + : >0 - limit task thread pool size to 'count' + -silent : do not output running test count + -noshadow : do not shadow copy assemblies + -teamcity : forces TeamCity mode (normally auto-detected) + -appveyor : forces AppVeyor CI mode (normally auto-detected) + -wait : wait for input after completion + -trait "name=value" : only run tests with matching name/value traits + : if specified more than once, acts as an OR operation + -notrait "name=value" : do not run tests with matching name/value traits + : if specified more than once, acts as an AND operation + -xml : output results to xUnit.net v2 style XML file + -xmlv1 : output results to xUnit.net v1 style XML file + -html : output results to HTML file +*) + +type ParallelOption = + | None = 0 + | Collections = 1 + | Assemblies = 2 + | All = 3 + +/// Option which allows to specify if an xUnit error should break the build. +type XUnit2ErrorLevel = TestRunnerErrorLevel // a type alias to keep backwards compatibility + +/// The xUnit parameter type +type XUnit2Params = + { /// The path to the xunit.console.exe - FAKE will scan all subfolders to find it automatically. + ToolPath : string + /// The file name of the config file (optional). + ConfigFile : string + /// set parallelization based on option + /// none - turn off all parallelization + /// collections - only parallelize collections + /// assemblies - only parallelize assemblies + /// all - parallelize assemblies & collections + Parallel : ParallelOption + /// maximum thread count for collection parallelization + /// 0 - run with unbounded thread count + /// >0 - limit task thread pool size to 'count' + MaxThreads : int + /// Output running test count + Silent : bool + /// Shadow copy + ShadowCopy : bool + /// forces TeamCity mode (normally auto-detected) + Teamcity : bool + /// forces AppVeyor CI mode (normally auto-detected) + Appveyor : bool + // wait for input after completion + Wait : bool + /// The working directory (optional). + WorkingDir : string + /// If the timeout is reached the xUnit task will be killed. Default is 5 minutes. + TimeOut : TimeSpan + /// Test runner error level. Option which allows to specify if an xUnit error should break the build. + ErrorLevel : XUnit2ErrorLevel + /// Include named traits with comma separated values + IncludeTraits : (string * string) option + /// Exclude named traits with comma separated values + ExcludeTraits : (string * string) option + /// output results to xUnit.net v2 style XML file + XmlOutput : bool + /// output results to xUnit.net v1 style XML file + XmlOutputV1 : bool + /// output results to HTML file + HtmlOutput : bool + /// output directory + OutputDir : string } + +/// The xUnit default parameters +let empty2Trait : (string * string) option = None + +let XUnit2Defaults = + { ToolPath = findToolInSubPath "xunit.console.exe" (currentDirectory @@ "tools" @@ "xUnit") + ConfigFile = null + Parallel = ParallelOption.None + MaxThreads = 0 + Silent = false + ShadowCopy = true + Teamcity = false + Appveyor = false + Wait = false + WorkingDir = null + TimeOut = TimeSpan.FromMinutes 5. + ErrorLevel = Error + IncludeTraits = empty2Trait + ExcludeTraits = empty2Trait + XmlOutput = false + XmlOutputV1 = false + HtmlOutput = false + OutputDir = null } + +/// Builds the command line arguments from the given parameter record and the given assemblies. +/// [omit] +let buildXUnit2Args parameters assembly = + let fi = fileInfo assembly + let name = fi.Name + + let dir = + if isNullOrEmpty parameters.OutputDir then String.Empty + else Path.GetFullPath parameters.OutputDir + + let traits includeExclude (name, values : string) = + values.Split([| ',' |], System.StringSplitOptions.RemoveEmptyEntries) + |> Seq.collect (fun value -> + [| includeExclude + sprintf "\"%s=%s\"" name value |]) + |> String.concat " " + + let parallelOptionsText = + parameters.Parallel.ToString().ToLower() + + new StringBuilder() + |> appendFileNamesIfNotNull [ assembly ] + |> append "-parallel" + |> append (sprintf "%s" parallelOptionsText) + |> append "-maxthreads" + |> append (sprintf "%i" parameters.MaxThreads) + |> appendIfFalse parameters.ShadowCopy "-noshadow" + |> appendIfTrue (buildServer = TeamCity) "-teamcity" + |> appendIfTrue (buildServer = AppVeyor) "-appveyor" + |> appendIfTrue parameters.Wait "-wait" + |> appendIfTrue parameters.Silent "-silent" + |> appendIfTrue parameters.XmlOutput (sprintf "-xml\" \"%s" (dir @@ (name + ".xml"))) + |> appendIfTrue parameters.XmlOutputV1 (sprintf "-xmlv1\" \"%s" (dir @@ (name + ".xml"))) + |> appendIfTrue parameters.HtmlOutput (sprintf "-html\" \"%s" (dir @@ (name + ".html"))) + |> appendIfSome parameters.IncludeTraits (traits "-trait") + |> appendIfSome parameters.ExcludeTraits (traits "-notrait") + |> toText + + +/// Runs xUnit unit tests in the given assemblies via the given xUnit runner. +/// Will fail if the runner terminates with non-zero exit code for any of the assemblies. +/// Offending assemblies will be listed in the error message. +/// +/// The xUnit runner terminates with a non-zero exit code if any of the tests +/// in the given assembly fail. +/// ## Parameters +/// +/// - `setParams` - Function used to manipulate the default XUnitParams value. +/// - `assemblies` - Sequence of one or more assemblies containing xUnit unit tests. +/// +/// ## Sample usage +/// +/// Target "Test" (fun _ -> +/// !! (testDir + @"\xUnit.Test.*.dll") +/// |> xUnit (fun p -> {p with OutputDir = testDir }) +/// ) +let xUnit2 setParams assemblies = + let details = separated ", " assemblies + traceStartTask "xUnit2" details + let parameters = setParams XUnit2Defaults + + let runTests assembly = + let args = buildXUnit2Args parameters assembly + 0 = ExecProcess (fun info -> + info.FileName <- parameters.ToolPath + info.WorkingDirectory <- parameters.WorkingDir + info.Arguments <- args) parameters.TimeOut + + let failedTests = + [ for asm in List.ofSeq assemblies do + if runTests asm |> not then yield asm ] + + if not (List.isEmpty failedTests) then + sprintf "xUnit2 failed for the following assemblies: %s" (separated ", " failedTests) + |> match parameters.ErrorLevel with + | Error | FailOnFirstError -> failwith + | DontFailBuild -> traceImportant + traceEndTask "xUnit2" details \ No newline at end of file diff --git a/src/test/Test.FAKECore/Test.FAKECore.csproj b/src/test/Test.FAKECore/Test.FAKECore.csproj index c8b4c9add90..0c6b037695e 100644 --- a/src/test/Test.FAKECore/Test.FAKECore.csproj +++ b/src/test/Test.FAKECore/Test.FAKECore.csproj @@ -121,6 +121,7 @@ + diff --git a/src/test/Test.FAKECore/XUnit2HelperSpecs.cs b/src/test/Test.FAKECore/XUnit2HelperSpecs.cs new file mode 100644 index 00000000000..8b2040d4393 --- /dev/null +++ b/src/test/Test.FAKECore/XUnit2HelperSpecs.cs @@ -0,0 +1,29 @@ +using System; +using Fake; +using Machine.Specifications; +using Microsoft.FSharp.Core; + +namespace Test.FAKECore +{ + public class XUnit2HelperSpecs + { + It args_should_not_include_traits = () => BuildXUnit2Args(XUnit2Helper.empty2Trait, XUnit2Helper.empty2Trait).ShouldNotContain("trait"); + It args_should_include_one_trait = () => BuildXUnit2Args(Trait("name", "value"), XUnit2Helper.empty2Trait).ShouldContain(@" -trait ""name=value"""); + It args_should_include_two_traits = () => BuildXUnit2Args(Trait("name", "value1,value2"), XUnit2Helper.empty2Trait).ShouldContain(@" -trait ""name=value1"" -trait ""name=value2"""); + It args_should_exclude_one_trait = () => BuildXUnit2Args(XUnit2Helper.empty2Trait, Trait("name", "value")).ShouldContain(@"-notrait ""name=value"""); + It args_should_exclude_two_traits = () => BuildXUnit2Args(XUnit2Helper.empty2Trait, Trait("name", "value1,value2")).ShouldContain(@" -notrait ""name=value1"" -notrait ""name=value2"""); + It args_should_contain_paraller = () => BuildXUnit2Args(XUnit2Helper.empty2Trait, XUnit2Helper.empty2Trait).ShouldContain(@"-parallel"); + + private static FSharpOption> Trait(string name, string values) + { + return new FSharpOption>(new Tuple(name,values)); + } + + private static string BuildXUnit2Args(FSharpOption> includeTrait, FSharpOption> excludeTrait) + { + var parameters = new XUnit2Helper.XUnit2Params("", "", XUnit2Helper.ParallelOption.None, 0, false, false, + true, false, false,null, TimeSpan.FromMinutes(5), UnitTestCommon.TestRunnerErrorLevel.Error, includeTrait, excludeTrait, false, false, true, ""); + return XUnit2Helper.buildXUnit2Args(parameters, "test.dll"); + } + } +}