diff --git a/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj b/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj index 0a2421f3262..e689cf1bf39 100644 --- a/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj +++ b/tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj @@ -11,6 +11,9 @@ + + XunitSetup.fs + diff --git a/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs b/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs index 1e5187167a7..70ca6baa84e 100644 --- a/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs +++ b/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs @@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation = | CompilationResult.Success ({OutputPath = Some dllFile} as s) -> let fsharpCoreFile = typeof>.Assembly.Location File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true) - let _exitCode, _stdout, stderr, _exn = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true) + let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true) - Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow") + Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow") | _ -> failwith (sprintf "%A" compilationResult) diff --git a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj index 02b1c629b91..ffc935a2075 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj +++ b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj @@ -28,6 +28,9 @@ + + XunitSetup.fs + diff --git a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs index 3f17e0cfa11..5ab961176f5 100644 --- a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs +++ b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs @@ -31,12 +31,13 @@ module ScriptRunner = let cu = cu |> withDefines defaultDefines match cu with | FS fsSource -> + use capture = new TestConsole.ExecutionCapture() let engine = createEngine (fsSource.Options |> Array.ofList,version) let res = evalScriptFromDiskInSharedSession engine cu match res with | CompilationResult.Failure _ -> res | CompilationResult.Success s -> - if engine.GetOutput().Contains "TEST PASSED OK" then + if capture.OutText |> TestFramework.outputPassed then res else failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}" diff --git a/tests/FSharp.Compiler.ComponentTests/xunit.runner.json b/tests/FSharp.Compiler.ComponentTests/xunit.runner.json index 2d07715ae5f..d1866b85c61 100644 --- a/tests/FSharp.Compiler.ComponentTests/xunit.runner.json +++ b/tests/FSharp.Compiler.ComponentTests/xunit.runner.json @@ -1,7 +1,6 @@ { - "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "ifAvailable", - "shadowCopy": false, - "parallelizeTestCollections": false, - "maxParallelThreads": 1 + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "appDomain": "denied", + "parallelizeTestCollections": false, + "maxParallelThreads": 1 } diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs index cc027d098af..20c5919e00e 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs @@ -13,6 +13,7 @@ open FSharp.Compiler.DependencyManager open FSharp.Compiler.Diagnostics open FSharp.DependencyManager.Nuget open FSharp.Test.ScriptHelpers +open FSharp.Test open FSharp.Test.Utilities open Internal.Utilities @@ -148,8 +149,7 @@ type DependencyManagerInteractiveTests() = Assert.Equal(0, result.Roots |> Seq.length) () - - [] + [] member _.``Multiple Instances of DependencyProvider should be isolated``() = let assemblyProbingPaths () = Seq.empty @@ -721,101 +721,65 @@ x |> Seq.iter(fun r -> [] member _.``Verify that #help produces help text for fsi + dependency manager``() = - let expected = [| - """ F# Interactive directives:""" - """""" - """ #r "file.dll";; // Reference (dynamically load) the given DLL""" - """ #i "package source uri";; // Include package source uri when searching for packages""" - """ #I "path";; // Add the given search path for referenced DLLs""" - """ #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced""" - """ #time ["on"|"off"];; // Toggle timing on/off""" - """ #clear;; // Clear screen""" - """ #help;; // Display help""" - """ #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;""" - """ #quit;; // Exit""" - """""" - """ F# Interactive command line options:""" - """""" - - // this is the end of the line each different platform has a different mechanism for starting fsi - // Actual output looks similar to: """ See 'testhost --help' for options""" - """--help' for options""" - - """""" - """""" - |] + let expected = """ + F# Interactive directives: + + #r "file.dll";; // Reference (dynamically load) the given DLL + #i "package source uri";; // Include package source uri when searching for packages + #I "path";; // Add the given search path for referenced DLLs + #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced + #time ["on"|"off"];; // Toggle timing on/off + #help;; // Display help + #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";; + #clear;; // Clear screen + #quit;; // Exit + + F# Interactive command line options:""" - let mutable found = 0 - let lines = System.Collections.Generic.List() - use sawExpectedOutput = new ManualResetEvent(false) - let verifyOutput (line: string) = - let compareLine (s: string) = - if s = "" then line = "" - else line.EndsWith(s) - lines.Add(line) - match expected |> Array.tryFind(compareLine) with - | None -> () - | Some t -> - found <- found + 1 - if found = expected.Length then sawExpectedOutput.Set() |> ignore - - let text = "#help" use script = new FSharpScript(quiet = false, langVersion = LangVersion.V47) - let mutable found = 0 - script.OutputProduced.Add (fun line -> verifyOutput line) - let opt = script.Eval(text) |> getValue - Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines) + use capture = new TestConsole.ExecutionCapture() + let opt = script.Eval("#help") |> getValue + + let output = capture.OutText + + Assert.Contains(expected, output) + + // this is the end of the line each different platform has a different mechanism for starting fsi + // Actual output looks similar to: """ See 'testhost --help' for options""" + Assert.EndsWith("--help' for options", output.Trim()) [] member _.``Verify that #help produces help text for fsi + dependency manager language version preview``() = - let expected = [| - """ F# Interactive directives:""" - """""" - """ #r "file.dll";; // Reference (dynamically load) the given DLL""" - """ #i "package source uri";; // Include package source uri when searching for packages""" - """ #I "path";; // Add the given search path for referenced DLLs""" - """ #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced""" - """ #time ["on"|"off"];; // Toggle timing on/off""" - """ #help;; // Display help""" - """ #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;""" - """ #r "nuget:FSharp.Data, 3.1.2";; // Load Nuget Package 'FSharp.Data' version '3.1.2'""" - """ #r "nuget:FSharp.Data";; // Load Nuget Package 'FSharp.Data' with the highest version""" - """ #clear;; // Clear screen""" - """ #quit;; // Exit""" - """""" - """ F# Interactive command line options:""" - """""" - - // this is the end of the line each different platform has a different mechanism for starting fsi - // Actual output looks similar to: """ See 'testhost --help' for options""" - """--help' for options""" - - """""" - """""" - |] + let expected = """ + F# Interactive directives: + + #r "file.dll";; // Reference (dynamically load) the given DLL + #i "package source uri";; // Include package source uri when searching for packages + #I "path";; // Add the given search path for referenced DLLs + #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced + #time ["on"|"off"];; // Toggle timing on/off + #help;; // Display help + #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";; + #r "nuget:FSharp.Data, 3.1.2";; // Load Nuget Package 'FSharp.Data' version '3.1.2' + #r "nuget:FSharp.Data";; // Load Nuget Package 'FSharp.Data' with the highest version + #clear;; // Clear screen + #quit;; // Exit + + F# Interactive command line options:""" - let mutable found = 0 - let lines = System.Collections.Generic.List() - use sawExpectedOutput = new ManualResetEvent(false) - let verifyOutput (line: string) = - let compareLine (s: string) = - if s = "" then line = "" - else line.EndsWith(s) - lines.Add(line) - match expected |> Array.tryFind(compareLine) with - | None -> () - | Some t -> - found <- found + 1 - if found = expected.Length then sawExpectedOutput.Set() |> ignore - - let text = "#help" use script = new FSharpScript(quiet = false, langVersion = LangVersion.Preview) - let mutable found = 0 - script.OutputProduced.Add (fun line -> verifyOutput line) - let opt = script.Eval(text) |> getValue - Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines) + use capture = new TestConsole.ExecutionCapture() + let opt = script.Eval("#help") |> getValue + + let output = capture.OutText + + Assert.Contains(expected, output) + + // this is the end of the line each different platform has a different mechanism for starting fsi + // Actual output looks similar to: """ See 'testhost --help' for options""" + Assert.EndsWith("--help' for options", output.Trim()) [] member _.``Verify that timeout --- times out and fails``() = diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj index 3bf2d528a4f..a3ac4bd6d1a 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj @@ -12,6 +12,9 @@ + + XunitSetup.fs + diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs index bf3a9cbaac6..d085005d9ac 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs @@ -3,6 +3,7 @@ namespace FSharp.Compiler.Scripting.UnitTests open System +open System.Text open System.Diagnostics open System.IO open System.Reflection @@ -11,6 +12,7 @@ open System.Threading open System.Threading.Tasks open FSharp.Compiler.Interactive open FSharp.Compiler.Interactive.Shell +open FSharp.Test open FSharp.Test.ScriptHelpers open Xunit @@ -87,7 +89,8 @@ x [] member _.``Capture console input``() = - use script = new FSharpScript(input = "stdin:1234\r\n") + use _ = new TestConsole.ProvideInput("stdin:1234\r\n") + use script = new FSharpScript() let opt = script.Eval("System.Console.ReadLine()") |> getValue let value = opt.Value Assert.Equal(typeof, value.ReflectionType) @@ -95,14 +98,11 @@ x [] member _.``Capture console output/error``() = + use capture = new TestConsole.ExecutionCapture() use script = new FSharpScript() - use sawOutputSentinel = new ManualResetEvent(false) - use sawErrorSentinel = new ManualResetEvent(false) - script.OutputProduced.Add (fun line -> if line = "stdout:1234" then sawOutputSentinel.Set() |> ignore) - script.ErrorProduced.Add (fun line -> if line = "stderr:5678" then sawErrorSentinel.Set() |> ignore) script.Eval("printfn \"stdout:1234\"; eprintfn \"stderr:5678\"") |> ignoreValue - Assert.True(sawOutputSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see output sentinel value written") - Assert.True(sawErrorSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see error sentinel value written") + Assert.Contains("stdout:1234", capture.OutText) + Assert.Contains("stderr:5678", capture.ErrorText) [] member _.``Maintain state between submissions``() = @@ -306,30 +306,26 @@ printfn "{@"%A"}" result [] member _.``Eval script with invalid PackageName should fail immediately``() = + use capture = new TestConsole.ExecutionCapture() use script = new FSharpScript(additionalArgs=[| |]) - let mutable found = 0 - let outp = System.Collections.Generic.List() - script.OutputProduced.Add( - fun line -> - if line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package") then - found <- found + 1 - outp.Add(line)) let result, errors = script.Eval("""#r "nuget:FSharp.Really.Not.A.Package" """) - Assert.True( (found = 0), "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'") + + let lines = capture.OutText.Split([| Environment.NewLine |], StringSplitOptions.None) + let found = lines |> Seq.exists (fun line -> line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package")) + Assert.False(found, "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'") Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("error NU1101:")), "Expect to error containing 'error NU1101:'") Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")), "Expect to error containing 'FSharp.Really.Not.A.Package'") [] member _.``Eval script with invalid PackageName should fail immediately and resolve one time only``() = + use capture = new TestConsole.ExecutionCapture() use script = new FSharpScript(additionalArgs=[| |]) - let mutable foundResolve = 0 - script.OutputProduced.Add (fun line -> if line.Contains("error NU1101:") then foundResolve <- foundResolve + 1) let result, errors = script.Eval(""" #r "nuget:FSharp.Really.Not.A.Package" #r "nuget:FSharp.Really.Not.Another.Package" """) - Assert.True( (foundResolve = 0), (sprintf "Did not expected to see 'error NU1101:' in output" )) + Assert.DoesNotContain("error NU1101:", capture.OutText) Assert.Equal(2, (errors |> Seq.filter (fun error -> error.Message.Contains("error NU1101:")) |> Seq.length)) Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")) |> Seq.length)) Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.Another.Package")) |> Seq.length)) @@ -479,6 +475,7 @@ let x = script.Eval(code) |> ignoreValue Assert.False(foundInner) + [] member _.``Script with nuget package that yields out of order dependencies works correctly``() = // regression test for: https://github.com/dotnet/fsharp/issues/9217 diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json b/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json index 2d07715ae5f..b180883cc0f 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json @@ -1,7 +1,6 @@ { "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "ifAvailable", - "shadowCopy": false, + "appDomain": "denied", "parallelizeTestCollections": false, "maxParallelThreads": 1 } diff --git a/tests/FSharp.Compiler.Service.Tests/Common.fs b/tests/FSharp.Compiler.Service.Tests/Common.fs index 5fd97f5bb21..986e2be5132 100644 --- a/tests/FSharp.Compiler.Service.Tests/Common.fs +++ b/tests/FSharp.Compiler.Service.Tests/Common.fs @@ -476,7 +476,7 @@ let assertRange [] module TempDirUtils = let getTempPath dir = - Path.Combine(TestFramework.tempDirectoryOfThisTestRun, dir) + Path.Combine(tempDirectoryOfThisTestRun.Value.FullName, dir) /// Returns the file name part of a temp file name created with tryCreateTemporaryFileName () /// and an added process id and thread id to ensure uniqueness between threads. diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj index 910e9543ada..fc64abb4138 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj @@ -21,6 +21,9 @@ + + XunitSetup.fs + diff --git a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs index ac563c3e1ed..dfce65ef186 100644 --- a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs @@ -4401,7 +4401,7 @@ let ``Test Project33 extension methods`` () = ("GetValue", ["member"; "extmem"])] module internal Project34 = - let directoryPath = createTemporaryDirectory "Project34" + let directoryPath = createTemporaryDirectory().FullName let sourceFileName = Path.Combine(directoryPath, "Program.fs") let dllName = Path.ChangeExtension(sourceFileName, ".dll") let projFileName = Path.ChangeExtension(sourceFileName, ".fsproj") diff --git a/tests/FSharp.Compiler.Service.Tests/xunit.runner.json b/tests/FSharp.Compiler.Service.Tests/xunit.runner.json index 743febb7028..2b65d19ea3e 100644 --- a/tests/FSharp.Compiler.Service.Tests/xunit.runner.json +++ b/tests/FSharp.Compiler.Service.Tests/xunit.runner.json @@ -1,5 +1,4 @@ { - "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "ifAvailable", - "shadowCopy": false + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "appDomain": "denied" } diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj index 2ba4f6f837d..305d234a837 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj +++ b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj @@ -26,6 +26,9 @@ + + XunitSetup.fs + diff --git a/tests/FSharp.Core.UnitTests/xunit.runner.json b/tests/FSharp.Core.UnitTests/xunit.runner.json index 2d07715ae5f..d1866b85c61 100644 --- a/tests/FSharp.Core.UnitTests/xunit.runner.json +++ b/tests/FSharp.Core.UnitTests/xunit.runner.json @@ -1,7 +1,6 @@ { - "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "ifAvailable", - "shadowCopy": false, - "parallelizeTestCollections": false, - "maxParallelThreads": 1 + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "appDomain": "denied", + "parallelizeTestCollections": false, + "maxParallelThreads": 1 } diff --git a/tests/FSharp.Test.Utilities/Compiler.fs b/tests/FSharp.Test.Utilities/Compiler.fs index 5e5629b089f..115964702f8 100644 --- a/tests/FSharp.Test.Utilities/Compiler.fs +++ b/tests/FSharp.Test.Utilities/Compiler.fs @@ -170,16 +170,16 @@ module rec Compiler = Message: string SubCategory: string } - // This type is used either for the output of the compiler (typically in CompilationResult coming from 'compile') - // or for the output of the code generated by the compiler (in CompilationResult coming from 'run') - type ExecutionOutput = - { ExitCode: int option - StdOut: string - StdErr: string - Exn: exn option } +// This type is used either for the output of the compiler (typically in CompilationResult coming from 'compile') +// or for the output of the code generated by the compiler (in CompilationResult coming from 'run') + + type EvalOutput = + { Result: Result + StdOut: string + StdErr: string } type RunOutput = - | EvalOutput of Result + | EvalOutput of EvalOutput | ExecutionOutput of ExecutionOutput type SourceCodeFileName = string @@ -701,20 +701,23 @@ module rec Compiler = let private compileFSharpCompilation compilation ignoreWarnings (cUnit: CompilationUnit) : CompilationResult = - use redirect = new RedirectConsole() + use capture = new TestConsole.ExecutionCapture() + let ((err: FSharpDiagnostic[], exn, outputFilePath: string), deps) = CompilerAssert.CompileRaw(compilation, ignoreWarnings) // Create and stash the console output let diagnostics = err |> fromFSharpDiagnostic + let outcome = exn |> Option.map Failure |> Option.defaultValue NoExitCode + let result = { OutputPath = None Dependencies = deps Adjust = 0 PerFileErrors = diagnostics Diagnostics = diagnostics |> List.map snd - Output = Some (RunOutput.ExecutionOutput { ExitCode = None; StdOut = redirect.Output(); StdErr = redirect.ErrorOutput(); Exn = exn }) + Output = Some (RunOutput.ExecutionOutput { Outcome = outcome; StdOut = capture.OutText; StdErr = capture.ErrorText }) Compilation = cUnit } @@ -733,7 +736,7 @@ module rec Compiler = let outputDirectory = match fs.OutputDirectory with | Some di -> di - | None -> DirectoryInfo(createTemporaryDirectory "compileFSharp") + | None -> createTemporaryDirectory() let references = processReferences fs.References outputDirectory let compilation = Compilation.CreateFromSources([fs.Source] @ fs.AdditionalSources, output, options, fs.TargetFramework, references, name, outputDirectory) compileFSharpCompilation compilation fs.IgnoreWarnings (FS fs) @@ -783,7 +786,7 @@ module rec Compiler = let outputDirectory = match csSource.OutputDirectory with | Some di -> di - | None -> DirectoryInfo(createTemporaryDirectory "compileCSharp") + | None -> createTemporaryDirectory() let additionalReferences = processReferences csSource.References outputDirectory @@ -921,20 +924,18 @@ module rec Compiler = let fileName = fsSource.Source.ChangeExtension.GetSourceFileName let references = - let disposals = ResizeArray() let outputDirectory = match fsSource.OutputDirectory with | Some di -> di - | None -> DirectoryInfo(createTemporaryDirectory "typecheckResults") + | None -> createTemporaryDirectory() let references = processReferences fsSource.References outputDirectory if references.IsEmpty then Array.empty else outputDirectory.Create() - disposals.Add({ new IDisposable with member _.Dispose() = outputDirectory.Delete(true) }) // Note that only the references are relevant here let compilation = Compilation.Compilation([], CompileOutput.Exe,Array.empty, TargetFramework.Current, references, None, None) - evaluateReferences outputDirectory disposals fsSource.IgnoreWarnings compilation + evaluateReferences outputDirectory fsSource.IgnoreWarnings compilation |> fst let options = @@ -981,19 +982,17 @@ module rec Compiler = | SourceCodeFileKind.Fsx _ -> true | _ -> false | _ -> false - let exitCode, output, errors, exn = CompilerAssert.ExecuteAndReturnResult (p, isFsx, s.Dependencies, false) - printfn "---------output-------\n%s\n-------" output - printfn "---------errors-------\n%s\n-------" errors - let executionResult = { s with Output = Some (ExecutionOutput { ExitCode = exitCode; StdOut = output; StdErr = errors; Exn = exn }) } - match exn with - | None -> CompilationResult.Success executionResult - | Some _ -> CompilationResult.Failure executionResult + let output = CompilerAssert.ExecuteAndReturnResult (p, isFsx, s.Dependencies, false) + let executionResult = { s with Output = Some (ExecutionOutput output) } + match output.Outcome with + | Failure _ -> CompilationResult.Failure executionResult + | _ -> CompilationResult.Success executionResult let compileAndRun = compile >> run let compileExeAndRun = asExe >> compileAndRun - let private processScriptResults fs (evalResult: Result, err: FSharpDiagnostic[]) = + let private processScriptResults fs (evalResult: Result, err: FSharpDiagnostic[]) outputWritten errorsWritten = let perFileDiagnostics = err |> fromFSharpDiagnostic let diagnostics = perFileDiagnostics |> List.map snd let (errors, warnings) = partitionErrors diagnostics @@ -1003,7 +1002,7 @@ module rec Compiler = Adjust = 0 Diagnostics = if fs.IgnoreWarnings then errors else diagnostics PerFileErrors = perFileDiagnostics - Output = Some (EvalOutput evalResult) + Output = Some (EvalOutput ({Result = evalResult; StdOut = outputWritten; StdErr = errorsWritten})) Compilation = FS fs } let evalError = match evalResult with Ok _ -> false | _ -> true @@ -1015,7 +1014,10 @@ module rec Compiler = let private evalFSharp (fs: FSharpCompilationSource) (script:FSharpScript) : CompilationResult = let source = fs.Source.GetSourceText |> Option.defaultValue "" - script.Eval(source) |> (processScriptResults fs) + use capture = new TestConsole.ExecutionCapture() + let result = script.Eval(source) + let outputWritten, errorsWritten = capture.OutText, capture.ErrorText + processScriptResults fs result outputWritten errorsWritten let scriptingShim = Path.Combine(__SOURCE_DIRECTORY__,"ScriptingShims.fsx") let private evalScriptFromDisk (fs: FSharpCompilationSource) (script:FSharpScript) : CompilationResult = @@ -1027,7 +1029,10 @@ module rec Compiler = |> List.map (sprintf " @\"%s\"") |> String.Concat - script.Eval("#load " + fileNames ) |> (processScriptResults fs) + use capture = new TestConsole.ExecutionCapture() + let result = script.Eval("#load " + fileNames) + let outputWritten, errorsWritten = capture.OutText, capture.ErrorText + processScriptResults fs result outputWritten errorsWritten let eval (cUnit: CompilationUnit) : CompilationResult = match cUnit with @@ -1037,7 +1042,7 @@ module rec Compiler = evalFSharp fs script | _ -> failwith "Script evaluation is only supported for F#." - let getSessionForEval args version = new FSharpScript(additionalArgs=args,quiet=false,langVersion=version) + let getSessionForEval args version = new FSharpScript(additionalArgs=args,quiet=true,langVersion=version) let evalInSharedSession (script:FSharpScript) (cUnit: CompilationUnit) : CompilationResult = match cUnit with @@ -1052,58 +1057,51 @@ module rec Compiler = let runFsi (cUnit: CompilationUnit) : CompilationResult = match cUnit with | FS fs -> - let disposals = ResizeArray() - try - let source = fs.Source.GetSourceText |> Option.defaultValue "" - let name = fs.Name |> Option.defaultValue "unnamed" - let options = fs.Options |> Array.ofList - let outputDirectory = - match fs.OutputDirectory with - | Some di -> di - | None -> DirectoryInfo(createTemporaryDirectory "runFsi") - outputDirectory.Create() - disposals.Add({ new IDisposable with member _.Dispose() = outputDirectory.Delete(true) }) - - let references = processReferences fs.References outputDirectory - let cmpl = Compilation.Create(fs.Source, fs.OutputType, options, fs.TargetFramework, references, name, outputDirectory) - let _compilationRefs, _deps = evaluateReferences outputDirectory disposals fs.IgnoreWarnings cmpl - let options = - let opts = new ResizeArray(fs.Options) - - // For every built reference add a -I path so that fsi can find it easily - for reference in references do - match reference with - | CompilationReference( cmpl, _) -> - match cmpl with - | Compilation(_sources, _outputType, _options, _targetFramework, _references, _name, outputDirectory) -> - if outputDirectory.IsSome then - opts.Add($"-I:\"{(outputDirectory.Value.FullName)}\"") - | _ -> () - opts.ToArray() - let errors, stdOut = CompilerAssert.RunScriptWithOptionsAndReturnResult options source - - let mkResult output = - { OutputPath = None - Dependencies = [] - Adjust = 0 - Diagnostics = [] - PerFileErrors= [] - Output = Some output - Compilation = cUnit } - - if errors.Count = 0 then - let output = - ExecutionOutput { ExitCode = None; StdOut = stdOut; StdErr = ""; Exn = None } - CompilationResult.Success (mkResult output) - else - let err = (errors |> String.concat "\n").Replace("\r\n","\n") - let output = - ExecutionOutput {ExitCode = None; StdOut = String.Empty; StdErr = err; Exn = None } - CompilationResult.Failure (mkResult output) - - finally - disposals - |> Seq.iter (fun x -> x.Dispose()) + let source = fs.Source.GetSourceText |> Option.defaultValue "" + let name = fs.Name |> Option.defaultValue "unnamed" + let options = fs.Options |> Array.ofList + let outputDirectory = + match fs.OutputDirectory with + | Some di -> di + | None -> createTemporaryDirectory() + outputDirectory.Create() + + let references = processReferences fs.References outputDirectory + let cmpl = Compilation.Create(fs.Source, fs.OutputType, options, fs.TargetFramework, references, name, outputDirectory) + let _compilationRefs, _deps = evaluateReferences outputDirectory fs.IgnoreWarnings cmpl + let options = + let opts = new ResizeArray(fs.Options) + + // For every built reference add a -I path so that fsi can find it easily + for reference in references do + match reference with + | CompilationReference( cmpl, _) -> + match cmpl with + | Compilation(_sources, _outputType, _options, _targetFramework, _references, _name, outputDirectory) -> + if outputDirectory.IsSome then + opts.Add($"-I:\"{(outputDirectory.Value.FullName)}\"") + | _ -> () + opts.ToArray() + let errors, stdOut, stdErr = CompilerAssert.RunScriptWithOptionsAndReturnResult options source + + let mkResult output = + { OutputPath = None + Dependencies = [] + Adjust = 0 + Diagnostics = [] + PerFileErrors= [] + Output = Some output + Compilation = cUnit } + + if errors.Count = 0 then + let output = + ExecutionOutput { Outcome = NoExitCode; StdOut = stdOut; StdErr = stdErr } + CompilationResult.Success (mkResult output) + else + let err = (errors |> String.concat "\n").Replace("\r\n","\n") + let output = + ExecutionOutput {Outcome = NoExitCode; StdOut = String.Empty; StdErr = err } + CompilationResult.Failure (mkResult output) | _ -> failwith "FSI running only supports F#." @@ -1190,9 +1188,11 @@ Actual: | Some p -> match ILChecker.verifyILAndReturnActual [] p expected with | true, _, _ -> result - | false, errorMsg, _actualIL -> CompilationResult.Failure( {s with Output = Some (ExecutionOutput {ExitCode = None; StdOut = errorMsg; StdErr = ""; Exn = None })} ) - - | CompilationResult.Failure f -> failwith $"Result should be \"Success\" in order to get IL. Failure: {Environment.NewLine}{f}" + | false, errorMsg, _actualIL -> CompilationResult.Failure( {s with Output = Some (ExecutionOutput {Outcome = NoExitCode; StdOut = errorMsg; StdErr = "" })} ) + | CompilationResult.Failure f -> + printfn "Failure:" + printfn $"{f}" + failwith $"Result should be \"Success\" in order to get IL." let verifyIL = doILCheck ILChecker.checkIL @@ -1282,7 +1282,7 @@ Actual: | Some actual -> let expected = stripVersion (normalizeNewlines expected) if expected <> actual then - failwith $"""Output does not match expected: ------------{Environment.NewLine}{expected}{Environment.NewLine}Actual: ------------{Environment.NewLine}{actual}{Environment.NewLine}""" + failwith $"""Output does not match expected:{Environment.NewLine}{expected}{Environment.NewLine}Actual:{Environment.NewLine}{actual}{Environment.NewLine}""" else cResult @@ -1295,7 +1295,7 @@ Actual: | Some actual -> for item in expected do if not(actual.Contains(item)) then - failwith $"""Output does not match expected: ------------{Environment.NewLine}{item}{Environment.NewLine}Actual: ------------{Environment.NewLine}{actual}{Environment.NewLine}""" + failwith $"""Output does not match expected:{Environment.NewLine}{item}{Environment.NewLine}Actual:{Environment.NewLine}{actual}{Environment.NewLine}""" cResult type ImportScope = { Kind: ImportDefinitionKind; Name: string } @@ -1515,18 +1515,15 @@ Actual: match result with | CompilationResult.Success _ -> result | CompilationResult.Failure r -> - let message = - [ sprintf "Operation failed (expected to succeed).\n All errors:\n%A\n" r.Diagnostics - match r.Output with - | Some (ExecutionOutput output) -> - sprintf "----output-----\n%s\n----error-------\n%s\n----------" output.StdOut output.StdErr - | Some (EvalOutput (Result.Error exn) ) -> - sprintf "----script error-----\n%s\n----------" (exn.ToString()) - | Some (EvalOutput (Result.Ok fsiVal) ) -> - sprintf "----script output-----\n%A\n----------" (fsiVal) - | _ -> () ] - |> String.concat "\n" - failwith message + eprintfn "\nAll errors:" + r.Diagnostics |> Seq.iter (eprintfn "%A") + + match r.Output with + | Some (EvalOutput { Result = Result.Error ex }) + | Some (ExecutionOutput {Outcome = Failure ex }) -> + raise ex + | _ -> + failwithf "Operation failed (expected to succeed)." let shouldFail (result: CompilationResult) : CompilationResult = match result with @@ -1706,13 +1703,15 @@ Actual: | None -> failwith "Execution output is missing, cannot check exit code." | Some o -> match o with - | ExecutionOutput {ExitCode = Some exitCode} -> Assert.Equal(expectedExitCode, exitCode) + | ExecutionOutput {Outcome = ExitCode exitCode} -> Assert.Equal(expectedExitCode, exitCode) | _ -> failwith "Cannot check exit code on this run result." result let private checkOutputInOrder (category: string) (substrings: string list) (selector: ExecutionOutput -> string) (result: CompilationResult) : CompilationResult = match result.RunOutput with - | None -> failwith (sprintf "Execution output is missing cannot check \"%A\"" category) + | None -> + printfn "Execution output is missing cannot check \"%A\"" category + failwith "Execution output is missing." | Some o -> match o with | ExecutionOutput e -> @@ -1743,9 +1742,11 @@ Actual: let private assertEvalOutput (selector: FsiValue -> 'T) (value: 'T) (result: CompilationResult) : CompilationResult = match result.RunOutput with | None -> failwith "Execution output is missing cannot check value." - | Some (EvalOutput (Ok (Some e))) -> Assert.Equal<'T>(value, (selector e)) - | Some (EvalOutput (Ok None )) -> failwith "Cannot assert value of evaluation, since it is None." - | Some (EvalOutput (Result.Error ex)) -> raise ex + | Some (EvalOutput output) -> + match output.Result with + | Ok (Some e) -> Assert.Equal<'T>(value, (selector e)) + | Ok None -> failwith "Cannot assert value of evaluation, since it is None." + | Result.Error ex -> raise ex | Some _ -> failwith "Only 'eval' output is supported." result @@ -1775,7 +1776,9 @@ Actual: |> Array.filter (fun s -> s.Length > 0) if not (actual |> Array.contains expected) then - failwith ($"The following signature:\n%s{expected}\n\nwas not found in:\n" + (actual |> String.concat "\n")) + printfn $"The following signature:\n%s{expected}\n\nwas not found in:" + actual |> Array.iter (printfn "%s") + failwith "Expected signature was not found." let private printSignaturesImpl pageWidth cUnit = cUnit diff --git a/tests/FSharp.Test.Utilities/CompilerAssert.fs b/tests/FSharp.Test.Utilities/CompilerAssert.fs index 37a8e25e164..202f7704a9d 100644 --- a/tests/FSharp.Test.Utilities/CompilerAssert.fs +++ b/tests/FSharp.Test.Utilities/CompilerAssert.fs @@ -2,6 +2,8 @@ namespace FSharp.Test +open System.Threading + #nowarn "57" open System @@ -61,6 +63,16 @@ module AssemblyResolver = do addResolver() #endif +type ExecutionOutcome = + | NoExitCode + | ExitCode of int + | Failure of exn + +type ExecutionOutput = + { Outcome: ExecutionOutcome + StdOut: string + StdErr: string } + [] type ILVerifier (dllFilePath: string) = @@ -295,14 +307,13 @@ and Compilation = | n -> Some n Compilation(sources, output, options, targetFramework, cmplRefs, name, outputDirectory) - -module rec CompilerAssertHelpers = +module CompilerAssertHelpers = let UseTransparentCompiler = FSharp.Compiler.CompilerConfig.FSharpExperimentalFeaturesEnabledAutomatically || not (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TEST_TRANSPARENT_COMPILER"))) - let checker = FSharpChecker.Create(suggestNamesForErrors=true, useTransparentCompiler=UseTransparentCompiler) + let checker = FSharpChecker.Create(suggestNamesForErrors=true, useTransparentCompiler = UseTransparentCompiler) // Unlike C# whose entrypoint is always string[] F# can make an entrypoint with 0 args, or with an array of string[] let mkDefaultArgs (entryPoint:MethodBase) : obj[] = [| @@ -322,48 +333,65 @@ module rec CompilerAssertHelpers = else entryPoint let args = mkDefaultArgs entryPoint - captureConsoleOutputs (fun () -> entryPoint.Invoke(Unchecked.defaultof, args)) + + use capture = new TestConsole.ExecutionCapture() + let outcome = + try + match entryPoint.Invoke(Unchecked.defaultof, args) with + | :? int as rc -> ExitCode rc + | _ -> NoExitCode + with + | exn -> Failure exn + outcome, capture.OutText, capture.ErrorText #if NETCOREAPP - let executeBuiltApp assembly deps isFsx = + let executeBuiltApp assemblyPath deps isFsx = let ctxt = AssemblyLoadContext("ContextName", true) try ctxt.add_Resolving(fun ctxt name -> deps |> List.tryFind (fun (x: string) -> Path.GetFileNameWithoutExtension x = name.Name) |> Option.map ctxt.LoadFromAssemblyPath - |> Option.defaultValue null) + |> Option.toObj) - executeAssemblyEntryPoint (ctxt.LoadFromAssemblyPath assembly) isFsx + executeAssemblyEntryPoint (ctxt.LoadFromAssemblyPath assemblyPath) isFsx finally ctxt.Unload() #else type Worker () = inherit MarshalByRefObject() - member x.ExecuteTestCase assemblyPath (deps: string[]) isFsx = - AppDomain.CurrentDomain.add_AssemblyResolve(ResolveEventHandler(fun _ args -> - deps - |> Array.tryFind (fun (x: string) -> Path.GetFileNameWithoutExtension x = AssemblyName(args.Name).Name) - |> Option.bind (fun x -> if FileSystem.FileExistsShim x then Some x else None) - |> Option.map Assembly.LoadFile - |> Option.defaultValue null)) - + member x.ExecuteTestCase assemblyPath isFsx = + // Set console streams for the AppDomain. + TestConsole.install() let assembly = Assembly.LoadFrom assemblyPath executeAssemblyEntryPoint assembly isFsx - let adSetup = - let setup = new System.AppDomainSetup () - let directory = Path.GetDirectoryName(typeof.Assembly.Location) - setup.ApplicationBase <- directory - setup + let executeBuiltApp assembly dependecies isFsx = + let thisAssemblyDirectory = Path.GetDirectoryName(typeof.Assembly.Location) + let setup = AppDomainSetup(ApplicationBase = thisAssemblyDirectory) + let testCaseDomain = AppDomain.CreateDomain($"built app {assembly}", null, setup) + + testCaseDomain.add_AssemblyResolve(fun _ args -> + dependecies + |> List.tryFind (fun path -> Path.GetFileNameWithoutExtension path = AssemblyName(args.Name).Name) + |> Option.filter FileSystem.FileExistsShim + |> Option.map Assembly.LoadFile + |> Option.toObj + ) - let executeBuiltApp assembly deps = - let ad = AppDomain.CreateDomain((Guid()).ToString(), null, adSetup) let worker = - use _ = new AlreadyLoadedAppDomainResolver() - (ad.CreateInstanceFromAndUnwrap(typeof.Assembly.CodeBase, typeof.FullName)) :?> Worker - worker.ExecuteTestCase assembly (deps |> Array.ofList) + (testCaseDomain.CreateInstanceFromAndUnwrap(typeof.Assembly.CodeBase, typeof.FullName)) :?> Worker + + let outcome, output, errors = worker.ExecuteTestCase assembly isFsx + // Replay streams captured in appdomain. + printf $"{output}" + eprintf $"{errors}" + + AppDomain.Unload testCaseDomain + + outcome, output, errors + #endif let defaultProjectOptions (targetFramework: TargetFramework) = @@ -413,27 +441,12 @@ module rec CompilerAssertHelpers = errors, ex, outputFilePath let compileDisposable (outputDirectory:DirectoryInfo) isExe options targetFramework nameOpt (sources:SourceCodeFileKind list) = - let disposeFile path = - { - new IDisposable with - member _.Dispose() = - try File.Delete path with | _ -> () - } - let disposals = ResizeArray() - let disposeList = - { - new IDisposable with - member _.Dispose() = - for item in disposals do - item.Dispose() - } let name = match nameOpt with | Some name -> name | _ -> getTemporaryFileNameInDirectory outputDirectory.FullName let outputFilePath = Path.ChangeExtension (Path.Combine(outputDirectory.FullName, name), if isExe then ".exe" else ".dll") - disposals.Add(disposeFile outputFilePath) let sources = [ for item in sources do @@ -443,7 +456,6 @@ module rec CompilerAssertHelpers = let source = item.ChangeExtension let destFileName = Path.Combine(outputDirectory.FullName, Path.GetFileName(source.GetSourceFileName)) File.WriteAllText (destFileName, text) - disposals.Add(disposeFile destFileName) yield source.WithFileName(destFileName) | None -> // On Disk file @@ -451,15 +463,9 @@ module rec CompilerAssertHelpers = let source = item.ChangeExtension let destFileName = Path.Combine(outputDirectory.FullName, Path.GetFileName(source.GetSourceFileName)) File.Copy(sourceFileName, destFileName, true) - disposals.Add(disposeFile destFileName) yield source.WithFileName(destFileName) ] - try - disposeList, rawCompile outputFilePath isExe options targetFramework sources - with - | _ -> - disposeList.Dispose() - reraise() + rawCompile outputFilePath isExe options targetFramework sources let assertErrors libAdjust ignoreWarnings (errors: FSharpDiagnostic []) expectedErrors = let errorMessage (error: FSharpDiagnostic) = @@ -520,7 +526,29 @@ module rec CompilerAssertHelpers = finally try Directory.Delete(tempDir, true) with | _ -> () - let rec evaluateReferences (outputPath:DirectoryInfo) (disposals: ResizeArray) ignoreWarnings (cmpl: Compilation) : string[] * string list = + let rec compileCompilationAux outputDirectory ignoreWarnings (cmpl: Compilation) : (FSharpDiagnostic[] * exn option * string) * string list = + + let compilationRefs, deps = evaluateReferences outputDirectory ignoreWarnings cmpl + let isExe, sources, options, targetFramework, name = + match cmpl with + | Compilation(sources, output, options, targetFramework, _, name, _) -> + (match output with | Module -> false | Library -> false | Exe -> true), // isExe + sources, + options, + targetFramework, + name + + let res = compileDisposable outputDirectory isExe (Array.append options compilationRefs) targetFramework name sources + + let deps2 = + compilationRefs + |> Array.filter (fun x -> not (x.Contains("--staticlink"))) + |> Array.map (fun x -> x.Replace("-r:", String.Empty)) + |> List.ofArray + + res, (deps @ deps2) + + and evaluateReferences (outputPath:DirectoryInfo) ignoreWarnings (cmpl: Compilation) : string[] * string list = match cmpl with | Compilation(_, _, _, _, cmpls, _, _) -> let compiledRefs = @@ -528,14 +556,13 @@ module rec CompilerAssertHelpers = |> List.map (fun cmpl -> match cmpl with | CompilationReference (cmpl, staticLink) -> - compileCompilationAux outputPath disposals ignoreWarnings cmpl, staticLink + compileCompilationAux outputPath ignoreWarnings cmpl, staticLink | TestCompilationReference (cmpl) -> let fileName = match cmpl with | TestCompilation.CSharp c when not (String.IsNullOrWhiteSpace c.AssemblyName) -> c.AssemblyName | _ -> getTemporaryFileNameInDirectory outputPath.FullName let tmp = Path.Combine(outputPath.FullName, Path.ChangeExtension(fileName, ".dll")) - disposals.Add({ new IDisposable with member _.Dispose() = File.Delete tmp }) cmpl.EmitAsFile tmp (([||], None, tmp), []), false) @@ -559,38 +586,9 @@ module rec CompilerAssertHelpers = compilationRefs, deps - let compileCompilationAux outputDirectory (disposals: ResizeArray) ignoreWarnings (cmpl: Compilation) : (FSharpDiagnostic[] * exn option * string) * string list = - - let compilationRefs, deps = evaluateReferences outputDirectory disposals ignoreWarnings cmpl - let isExe, sources, options, targetFramework, name = - match cmpl with - | Compilation(sources, output, options, targetFramework, _, name, _) -> - (match output with | Module -> false | Library -> false | Exe -> true), // isExe - sources, - options, - targetFramework, - name - - let disposal, res = compileDisposable outputDirectory isExe (Array.append options compilationRefs) targetFramework name sources - disposals.Add(disposal) - - let deps2 = - compilationRefs - |> Array.filter (fun x -> not (x.Contains("--staticlink"))) - |> Array.map (fun x -> x.Replace("-r:", String.Empty)) - |> List.ofArray - - res, (deps @ deps2) - let compileCompilation ignoreWarnings (cmpl: Compilation) f = - let disposals = ResizeArray() - try - let outputDirectory = DirectoryInfo(createTemporaryDirectory "compileCompilation") - disposals.Add({ new IDisposable with member _.Dispose() = try File.Delete (outputDirectory.FullName) with | _ -> () }) - f (compileCompilationAux outputDirectory disposals ignoreWarnings cmpl) - finally - disposals - |> Seq.iter (fun x -> x.Dispose()) + let outputDirectory = createTemporaryDirectory() + f (compileCompilationAux outputDirectory ignoreWarnings cmpl) // NOTE: This function will not clean up all the compiled projects after itself. // The reason behind is so we can compose verification of test runs easier. @@ -599,47 +597,14 @@ module rec CompilerAssertHelpers = let outputDirectory = match cmpl with | Compilation(outputDirectory = Some outputDirectory) -> DirectoryInfo(outputDirectory.FullName) - | Compilation _ -> DirectoryInfo(createTemporaryDirectory "returnCompilation") + | Compilation _ -> createTemporaryDirectory() outputDirectory.Create() - compileCompilationAux outputDirectory (ResizeArray()) ignoreWarnings cmpl - - let captureConsoleOutputs (func: unit -> obj) = - let out = Console.Out - let err = Console.Error + compileCompilationAux outputDirectory ignoreWarnings cmpl - let stdout = StringBuilder () - let stderr = StringBuilder () + let unwrapException (ex: exn) = ex.InnerException |> Option.ofObj |> Option.map _.Message |> Option.defaultValue ex.Message - use outWriter = new StringWriter (stdout) - use errWriter = new StringWriter (stderr) - - let rc, exn = - try - try - Console.SetOut outWriter - Console.SetError errWriter - let rc = func() - match rc with - | :? int as rc -> Some rc, None - | _ -> None, None - with e -> - let errorMessage = if e.InnerException <> null then e.InnerException.ToString() else e.ToString() - stderr.Append errorMessage |> ignore - None, Some e - finally - Console.SetOut out - Console.SetError err - outWriter.Close() - errWriter.Close() - - rc, stdout.ToString(), stderr.ToString(), exn - - let executeBuiltAppAndReturnResult (outputFilePath: string) (deps: string list) isFsx : (int option * string * string * exn option) = - let rc, stdout, stderr, exn = executeBuiltApp outputFilePath deps isFsx - rc, stdout, stderr, exn - - let executeBuiltAppNewProcessAndReturnResult (outputFilePath: string) : (int * string * string) = + let executeBuiltAppNewProcess (outputFilePath: string) = #if !NETCOREAPP let fileName = outputFilePath let arguments = "" @@ -659,13 +624,12 @@ module rec CompilerAssertHelpers = }""" let runtimeconfigPath = Path.ChangeExtension(outputFilePath, ".runtimeconfig.json") File.WriteAllText(runtimeconfigPath, runtimeconfig) - use _disposal = - { new IDisposable with - member _.Dispose() = try File.Delete runtimeconfigPath with | _ -> () } #endif let timeout = 30000 - let exitCode, output, errors = Commands.executeProcess fileName arguments (Path.GetDirectoryName(outputFilePath)) timeout - (exitCode, output |> String.concat "\n", errors |> String.concat "\n") + let rc, output, errors = Commands.executeProcess fileName arguments (Path.GetDirectoryName(outputFilePath)) timeout + let output = String.Join(Environment.NewLine, output) + let errors = String.Join(Environment.NewLine, errors) + ExitCode rc, output, errors open CompilerAssertHelpers @@ -678,7 +642,7 @@ type CompilerAssert private () = if errors.Length > 0 then Assert.Fail (sprintf "Compile had warnings and/or errors: %A" errors) - executeBuiltApp outputExe [] false |> ignore + executeBuiltApp outputExe [] false ) static let compileLibraryAndVerifyILWithOptions options (source: SourceCodeFileKind) (f: ILVerifier -> unit) = @@ -691,7 +655,6 @@ type CompilerAssert private () = f (ILVerifier outputFilePath) ) - static let compileLibraryAndVerifyDebugInfoWithOptions options (expectedFile: string) (source: SourceCodeFileKind) = let options = [| yield! options; yield"--test:DumpDebugInfo" |] compile false options source (fun (errors, _, outputFilePath) -> @@ -714,6 +677,8 @@ Updated automatically, please check diffs in your pull request, changes must be """ ) + static member UseTransparentCompiler = UseTransparentCompiler + static member Checker = checker static member DefaultProjectOptions = defaultProjectOptions @@ -740,15 +705,14 @@ Updated automatically, please check diffs in your pull request, changes must be returnCompilation cmpl (defaultArg ignoreWarnings false) static member ExecuteAndReturnResult (outputFilePath: string, isFsx: bool, deps: string list, newProcess: bool) = - if not newProcess then - let entryPointReturnCode, deps, isFsx, exn = executeBuiltAppAndReturnResult outputFilePath deps isFsx - entryPointReturnCode, deps, isFsx, exn - else - let processExitCode, deps, isFsx = executeBuiltAppNewProcessAndReturnResult outputFilePath - Some processExitCode, deps, isFsx, None - + let outcome, output, errors = + if not newProcess then + executeBuiltApp outputFilePath deps isFsx + else + executeBuiltAppNewProcess outputFilePath + { Outcome = outcome; StdOut = output; StdErr = errors} - static member Execute(cmpl: Compilation, ?ignoreWarnings, ?beforeExecute, ?newProcess, ?onOutput) = + static member ExecuteAux(cmpl: Compilation, ?ignoreWarnings, ?beforeExecute, ?newProcess) = let copyDependenciesToOutputDir (outputFilePath:string) (deps: string list) = let outputDirectory = Path.GetDirectoryName(outputFilePath) @@ -760,21 +724,26 @@ Updated automatically, please check diffs in your pull request, changes must be let ignoreWarnings = defaultArg ignoreWarnings false let beforeExecute = defaultArg beforeExecute copyDependenciesToOutputDir let newProcess = defaultArg newProcess false - let onOutput = defaultArg onOutput (fun _ -> ()) compileCompilation ignoreWarnings cmpl (fun ((errors, _, outputFilePath), deps) -> assertErrors 0 ignoreWarnings errors [||] beforeExecute outputFilePath deps - if newProcess then - let (exitCode, output, errors) = executeBuiltAppNewProcessAndReturnResult outputFilePath - if exitCode <> 0 then - Assert.Fail errors - onOutput output + if newProcess then + executeBuiltAppNewProcess outputFilePath else - let _rc, _stdout, _stderr, exn = executeBuiltApp outputFilePath deps false - exn |> Option.iter raise) + executeBuiltApp outputFilePath deps false + ) + + static member Execute(cmpl: Compilation, ?ignoreWarnings, ?beforeExecute, ?newProcess) = + let outcome, _, _ = CompilerAssert.ExecuteAux(cmpl, ?ignoreWarnings = ignoreWarnings, ?beforeExecute = beforeExecute, ?newProcess = newProcess) + match outcome with + | ExitCode n when n <> 0 -> failwith $"Process exited with code {n}." + | Failure exn -> raise exn + | _ -> () + static member ExecutionHasOutput(cmpl: Compilation, expectedOutput: string) = - CompilerAssert.Execute(cmpl, newProcess = true, onOutput = (fun output -> Assert.Equal(expectedOutput, output))) + let _, output, _ = CompilerAssert.ExecuteAux(cmpl, newProcess = true) + Assert.Equal(expectedOutput, output) static member Pass (source: string) = let parseResults, fileAnswer = checker.ParseAndCheckFileInProject("test.fs", 0, SourceText.ofString source, defaultProjectOptions TargetFramework.Current) |> Async.RunImmediate @@ -954,7 +923,7 @@ Updated automatically, please check diffs in your pull request, changes must be } )) - let snapshot = FSharpProjectSnapshot.FromOptions(projectOptions, getFileSnapshot) |> Async.RunSynchronously + let snapshot = FSharpProjectSnapshot.FromOptions(projectOptions, getFileSnapshot) |> Async.RunImmediate checker.ParseAndCheckProject(snapshot) else @@ -1038,10 +1007,10 @@ Updated automatically, please check diffs in your pull request, changes must be | Choice2Of2 ex -> errorMessages.Add(ex.Message) | _ -> () - errorMessages, outStream.ToString() + errorMessages, string outStream, string errStream static member RunScriptWithOptions options (source: string) (expectedErrorMessages: string list) = - let errorMessages, _ = CompilerAssert.RunScriptWithOptionsAndReturnResult options source + let errorMessages, _, _ = CompilerAssert.RunScriptWithOptionsAndReturnResult options source if expectedErrorMessages.Length <> errorMessages.Count then Assert.Fail(sprintf "Expected error messages: %A \n\n Actual error messages: %A" expectedErrorMessages errorMessages) else diff --git a/tests/FSharp.Test.Utilities/DirectoryAttribute.fs b/tests/FSharp.Test.Utilities/DirectoryAttribute.fs index 1266bc295a8..f54faec596d 100644 --- a/tests/FSharp.Test.Utilities/DirectoryAttribute.fs +++ b/tests/FSharp.Test.Utilities/DirectoryAttribute.fs @@ -32,7 +32,7 @@ type DirectoryAttribute(dir: string) = | _ -> None let createCompilationUnit path (filename: string) = - let outputDirectoryPath = createTemporaryDirectory "dir" + let outputDirectoryPath = createTemporaryDirectory().FullName let sourceFilePath = normalizePathSeparator (path ++ filename) let fsBslFilePath = sourceFilePath + baselineSuffix + ".err.bsl" let ilBslFilePath = diff --git a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj index 7ccc5306751..52c1893073c 100644 --- a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj +++ b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj @@ -28,7 +28,9 @@ scriptlib.fsx + + @@ -42,6 +44,10 @@ + + + + diff --git a/tests/FSharp.Test.Utilities/ScriptHelpers.fs b/tests/FSharp.Test.Utilities/ScriptHelpers.fs index 688941934c9..f060cb91398 100644 --- a/tests/FSharp.Test.Utilities/ScriptHelpers.fs +++ b/tests/FSharp.Test.Utilities/ScriptHelpers.fs @@ -24,26 +24,7 @@ type LangVersion = | Latest | SupportsMl -type private EventedTextWriter() = - inherit TextWriter() - let sb = StringBuilder() - let sw = new StringWriter() - let lineWritten = Event() - member _.LineWritten = lineWritten.Publish - override _.Encoding = Encoding.UTF8 - override _.Write(c: char) = - if c = '\n' then - let line = - let v = sb.ToString() - if v.EndsWith("\r") then v.Substring(0, v.Length - 1) - else v - sb.Clear() |> ignore - sw.WriteLine line - lineWritten.Trigger(line) - else sb.Append(c) |> ignore - override _.ToString() = sw.ToString() - -type FSharpScript(?additionalArgs: string[], ?quiet: bool, ?langVersion: LangVersion, ?input: string) = +type FSharpScript(?additionalArgs: string[], ?quiet: bool, ?langVersion: LangVersion) = let additionalArgs = defaultArg additionalArgs [||] let quiet = defaultArg quiet true @@ -72,31 +53,12 @@ type FSharpScript(?additionalArgs: string[], ?quiet: bool, ?langVersion: LangVer let argv = Array.append baseArgs additionalArgs - let inReader = new StringReader(defaultArg input "") - let outWriter = new EventedTextWriter() - let errorWriter = new EventedTextWriter() - - let previousIn, previousOut, previousError = Console.In, Console.Out, Console.Error - - do - Console.SetIn inReader - Console.SetOut outWriter - Console.SetError errorWriter - let fsi = FsiEvaluationSession.Create (config, argv, stdin, stdout, stderr) member _.ValueBound = fsi.ValueBound member _.Fsi = fsi - member _.OutputProduced = outWriter.LineWritten - - member _.ErrorProduced = errorWriter.LineWritten - - member _.GetOutput() = string outWriter - - member _.GetErrorOutput() = string errorWriter - member this.Eval(code: string, ?cancellationToken: CancellationToken, ?desiredCulture: Globalization.CultureInfo) = let originalCulture = Thread.CurrentThread.CurrentCulture Thread.CurrentThread.CurrentCulture <- Option.defaultValue Globalization.CultureInfo.InvariantCulture desiredCulture @@ -127,9 +89,6 @@ type FSharpScript(?additionalArgs: string[], ?quiet: bool, ?langVersion: LangVer interface IDisposable with member this.Dispose() = ((this.Fsi) :> IDisposable).Dispose() - Console.SetIn previousIn - Console.SetOut previousOut - Console.SetError previousError [] module TestHelpers = @@ -141,4 +100,4 @@ module TestHelpers = | Ok(value) -> value | Error ex -> raise ex - let ignoreValue = getValue >> ignore + let ignoreValue x = getValue x |> ignore diff --git a/tests/FSharp.Test.Utilities/ScriptingShims.fsx b/tests/FSharp.Test.Utilities/ScriptingShims.fsx index 392d2b002b6..be9ce9142c5 100644 --- a/tests/FSharp.Test.Utilities/ScriptingShims.fsx +++ b/tests/FSharp.Test.Utilities/ScriptingShims.fsx @@ -2,13 +2,7 @@ namespace global [] module GlobalShims = - - let errorStringWriter = new System.IO.StringWriter() - let oldConsoleError = System.Console.Error - do System.Console.SetError(errorStringWriter) - let exit (code:int) = - System.Console.SetError(oldConsoleError) - if code=0 then + if code = 0 then () - else failwith $"Script called function 'exit' with code={code} and collected in stderr: {errorStringWriter.ToString()}" \ No newline at end of file + else failwith $"Script called function 'exit' with code={code}." diff --git a/tests/FSharp.Test.Utilities/TestConsole.fs b/tests/FSharp.Test.Utilities/TestConsole.fs new file mode 100644 index 00000000000..b28cd398408 --- /dev/null +++ b/tests/FSharp.Test.Utilities/TestConsole.fs @@ -0,0 +1,89 @@ +namespace FSharp.Test + +open System +open System.IO +open System.Text +open System.Threading + +module TestConsole = + + /// Redirects reads performed on different async execution contexts to the relevant TextReader held by AsyncLocal. + type private RedirectingTextReader() = + inherit TextReader() + let holder = AsyncLocal<_>() + do holder.Value <- TextReader.Null + + override _.Peek() = holder.Value.Peek() + override _.Read() = holder.Value.Read() + member _.Value = holder.Value + member _.Set (reader: TextReader) = holder.Value <- reader + + /// Redirects writes performed on different async execution contexts to the relevant TextWriter held by AsyncLocal. + type private RedirectingTextWriter() = + inherit TextWriter() + let holder = AsyncLocal() + do holder.Value <- TextWriter.Null + + override _.Encoding = Encoding.UTF8 + override _.Write(value: char) = holder.Value.Write(value) + override _.Write(value: string) = holder.Value.Write(value) + override _.WriteLine(value: string) = holder.Value.WriteLine(value) + member _.Value = holder.Value + member _.Set (writer: TextWriter) = holder.Value <- writer + + let private localIn = new RedirectingTextReader() + let private localOut = new RedirectingTextWriter() + let private localError = new RedirectingTextWriter() + + let install () = + Console.SetIn localIn + Console.SetOut localOut + Console.SetError localError + + // Taps into the redirected console stream. + type private CapturingWriter(redirecting: RedirectingTextWriter) as this = + inherit StringWriter() + let wrapped = redirecting.Value + do redirecting.Set this + override _.Encoding = Encoding.UTF8 + override _.Write(value: char) = wrapped.Write(value); base.Write(value) + override _.Write(value: string) = wrapped.Write(value); base.Write(value) + override _.WriteLine(value: string) = wrapped.WriteLine(value); base.Write(value) + override _.Dispose (disposing: bool) = + redirecting.Set wrapped + base.Dispose(disposing: bool) + + /// Captures console streams for the current async execution context. + /// Each simultaneously executing test case runs in a separate async execution context. + /// + /// Can be used to capture just a single compilation or eval as well as the whole test case execution output. + type ExecutionCapture() = + do + Console.Out.Flush() + Console.Error.Flush() + + let output = new CapturingWriter(localOut) + let error = new CapturingWriter(localError) + + member _.Dispose() = + output.Dispose() + error.Dispose() + + interface IDisposable with + member this.Dispose (): unit = this.Dispose() + + member _.OutText = + Console.Out.Flush() + string output + + member _.ErrorText = + Console.Error.Flush() + string error + + type ProvideInput(input: string) = + let oldIn = localIn.Value + do + new StringReader(input) |> localIn.Set + + interface IDisposable with + member this.Dispose (): unit = localIn.Set oldIn diff --git a/tests/FSharp.Test.Utilities/TestFramework.fs b/tests/FSharp.Test.Utilities/TestFramework.fs index dfde63f2352..ada5bf3363e 100644 --- a/tests/FSharp.Test.Utilities/TestFramework.fs +++ b/tests/FSharp.Test.Utilities/TestFramework.fs @@ -13,24 +13,24 @@ open Xunit.Sdk let getShortId() = Guid.NewGuid().ToString().[..7] -// Temporary directory is TempPath + "/FSharp.Test.Utilities/yyy-MM-dd-xxxxxxx/" +// Temporary directory is TempPath + "/FSharp.Test.Utilities/xxxxxxx/" let tempDirectoryOfThisTestRun = - let tempDir = Path.GetTempPath() - let today = DateTime.Now.ToString("yyyy-MM-dd") - DirectoryInfo(tempDir) - .CreateSubdirectory($"FSharp.Test.Utilities/{today}-{getShortId()}") - .FullName + let temp = Path.GetTempPath() + lazy DirectoryInfo(temp).CreateSubdirectory($"FSharp.Test.Utilities/{getShortId()}") -let createTemporaryDirectory (part: string) = - DirectoryInfo(tempDirectoryOfThisTestRun) - .CreateSubdirectory($"{part}-{getShortId()}") - .FullName +let cleanUpTemporaryDirectoryOfThisTestRun () = + if tempDirectoryOfThisTestRun.IsValueCreated then + try tempDirectoryOfThisTestRun.Value.Delete(true) with _ -> () + +let createTemporaryDirectory () = + tempDirectoryOfThisTestRun.Value + .CreateSubdirectory($"{getShortId()}") let getTemporaryFileName () = - (createTemporaryDirectory "temp") ++ $"tmp_{getShortId()}" + createTemporaryDirectory().FullName ++ getShortId() let getTemporaryFileNameInDirectory (directory: string) = - directory ++ $"tmp_{getShortId()}" + directory ++ getShortId() // Well, this function is AI generated. let rec copyDirectory (sourceDir: string) (destinationDir: string) (recursive: bool) = @@ -425,13 +425,12 @@ let logConfig (cfg: TestConfig) = log "PEVERIFY = %s" cfg.PEVERIFY log "---------------------------------------------------------------" -let checkOutputPassed (output: string) = - Assert.True(output.Contains "TEST PASSED OK", $"Output does not contain 'TEST PASSED OK':\n{output}") +let outputPassed (output: string) = output.Contains "TEST PASSED OK" let checkResultPassed result = match result with | CmdResult.ErrorLevel (msg1, err) -> Assert.Fail (sprintf "%s. ERRORLEVEL %d" msg1 err) - | CmdResult.Success output -> checkOutputPassed output + | CmdResult.Success output -> Assert.True(outputPassed output, "Output does not contain 'TEST PASSED OK'") let checkResult result = match result with @@ -474,11 +473,8 @@ let testConfig sourceDir (relativePathToTestFixture: string) = let cfg = suiteHelpers.Value let testFixtureFullPath = Path.GetFullPath(sourceDir ++ relativePathToTestFixture) - let description = relativePathToTestFixture.Split('\\', '/') |> String.concat "-" - - let tempTestRoot = createTemporaryDirectory description let tempTestDir = - DirectoryInfo(tempTestRoot) + createTemporaryDirectory() .CreateSubdirectory(relativePathToTestFixture) .FullName copyDirectory testFixtureFullPath tempTestDir true @@ -487,7 +483,7 @@ let testConfig sourceDir (relativePathToTestFixture: string) = let createConfigWithEmptyDirectory() = let cfg = suiteHelpers.Value - { cfg with Directory = createTemporaryDirectory "temp" } + { cfg with Directory = createTemporaryDirectory().FullName } type RedirectToType = | Overwrite of FilePath diff --git a/tests/FSharp.Test.Utilities/Utilities.fs b/tests/FSharp.Test.Utilities/Utilities.fs index 1a6d0ff60f8..484ee6049e2 100644 --- a/tests/FSharp.Test.Utilities/Utilities.fs +++ b/tests/FSharp.Test.Utilities/Utilities.fs @@ -40,21 +40,6 @@ type FactForDESKTOPAttribute() = // This file mimics how Roslyn handles their compilation references for compilation testing module Utilities = - type RedirectConsole() = - let oldStdOut = Console.Out - let oldStdErr = Console.Error - let newStdOut = new StringWriter() - let newStdErr = new StringWriter() - do Console.SetOut(newStdOut) - do Console.SetError(newStdErr) - member _.Output () = string newStdOut - - member _.ErrorOutput () =string newStdErr - - interface IDisposable with - member _.Dispose() = - Console.SetOut(oldStdOut) - Console.SetError(oldStdErr) type Async with static member RunImmediate (computation: Async<'T>, ?cancellationToken ) = @@ -69,20 +54,6 @@ module Utilities = cancellationToken) task.Result - /// Disposable type to implement a simple resolve handler that searches the currently loaded assemblies to see if the requested assembly is already loaded. - type AlreadyLoadedAppDomainResolver () = - let resolveHandler = - ResolveEventHandler(fun _ args -> - let assemblies = AppDomain.CurrentDomain.GetAssemblies() - let assembly = assemblies |> Array.tryFind(fun a -> String.Compare(a.FullName, args.Name,StringComparison.OrdinalIgnoreCase) = 0) - assembly |> Option.defaultValue Unchecked.defaultof - ) - do AppDomain.CurrentDomain.add_AssemblyResolve(resolveHandler) - - interface IDisposable with - member this.Dispose() = AppDomain.CurrentDomain.remove_AssemblyResolve(resolveHandler) - - [] type TargetFramework = | NetStandard20 diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs new file mode 100644 index 00000000000..cda74f7867c --- /dev/null +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -0,0 +1,23 @@ +namespace FSharp.Test + +open System +open Xunit.Sdk +open Xunit.Abstractions + +open TestFramework + +/// Installs console support for parallel test runs. +type FSharpXunitFramework(sink: IMessageSink) = + inherit XunitTestFramework(sink) + do + // Because xUnit v2 lacks assembly fixture, the next best place to ensure things get called + // right at the start of the test run is here in the constructor. + // This gets executed once per test assembly. + MessageSink.sinkWriter |> ignore + TestConsole.install() + + interface IDisposable with + member _.Dispose() = + cleanUpTemporaryDirectoryOfThisTestRun () + base.Dispose() + diff --git a/tests/FSharp.Test.Utilities/XunitSetup.fs b/tests/FSharp.Test.Utilities/XunitSetup.fs new file mode 100644 index 00000000000..fb43569b1d0 --- /dev/null +++ b/tests/FSharp.Test.Utilities/XunitSetup.fs @@ -0,0 +1,8 @@ +namespace FSharp.Test + +open Xunit + +module XUnitSetup = + + [] + do () diff --git a/tests/fsharp/FSharpSuite.Tests.fsproj b/tests/fsharp/FSharpSuite.Tests.fsproj index 6673da9f911..dc8fb5a164e 100644 --- a/tests/fsharp/FSharpSuite.Tests.fsproj +++ b/tests/fsharp/FSharpSuite.Tests.fsproj @@ -17,8 +17,8 @@ - - scriptlib.fsx + + XunitSetup.fs diff --git a/tests/fsharp/xunit.runner.json b/tests/fsharp/xunit.runner.json index 4e5a48343ec..c222a4167a8 100644 --- a/tests/fsharp/xunit.runner.json +++ b/tests/fsharp/xunit.runner.json @@ -1,5 +1,4 @@ { "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", - "appDomain": "ifAvailable", - "shadowCopy": false + "appDomain": "denied" } \ No newline at end of file diff --git a/tests/scripts/scriptlib.fsx b/tests/scripts/scriptlib.fsx index 853aecb496e..04b23a93fa8 100644 --- a/tests/scripts/scriptlib.fsx +++ b/tests/scripts/scriptlib.fsx @@ -10,6 +10,14 @@ open System.IO open System.Text open System.Diagnostics +module MessageSink = + let sinkWriter = +#if DEBUG + Console.Out +#else + TextWriter.Null +#endif + [] module Scripting = @@ -77,7 +85,7 @@ module Scripting = if Directory.Exists output then Directory.Delete(output, true) - let log format = printfn format + let log format = fprintfn MessageSink.sinkWriter format type FilePath = string