Skip to content

Commit

Permalink
TestConsole supporting parallel execution (dotnet#17993)
Browse files Browse the repository at this point in the history
  • Loading branch information
majocha authored Nov 13, 2024
1 parent ec4abb3 commit fb69e58
Show file tree
Hide file tree
Showing 29 changed files with 479 additions and 481 deletions.
3 changes: 3 additions & 0 deletions tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
</PropertyGroup>

<ItemGroup>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="WriteCodeFragmentTests.fs" />
<Compile Include="MapSourceRootsTests.fs" />
</ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation =
| CompilationResult.Success ({OutputPath = Some dllFile} as s) ->
let fsharpCoreFile = typeof<voption<_>>.Assembly.Location
File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true)
let _exitCode, _stdout, stderr, _exn = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)

Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow")
Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow")

| _ -> failwith (sprintf "%A" compilationResult)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
</PropertyGroup>

<ItemGroup>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="CompilerDirectives\Line.fs" />
<Compile Include="CompilerDirectives\Ifdef.fs" />
<Compile Include="CompilerDirectives\NonStringArgs.fs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,13 @@ module ScriptRunner =
let cu = cu |> withDefines defaultDefines
match cu with
| FS fsSource ->
use capture = new TestConsole.ExecutionCapture()
let engine = createEngine (fsSource.Options |> Array.ofList,version)
let res = evalScriptFromDiskInSharedSession engine cu
match res with
| CompilationResult.Failure _ -> res
| CompilationResult.Success s ->
if engine.GetOutput().Contains "TEST PASSED OK" then
if capture.OutText |> TestFramework.outputPassed then
res
else
failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}"
Expand Down
9 changes: 4 additions & 5 deletions tests/FSharp.Compiler.ComponentTests/xunit.runner.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "ifAvailable",
"shadowCopy": false,
"parallelizeTestCollections": false,
"maxParallelThreads": 1
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "denied",
"parallelizeTestCollections": false,
"maxParallelThreads": 1
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ open FSharp.Compiler.DependencyManager
open FSharp.Compiler.Diagnostics
open FSharp.DependencyManager.Nuget
open FSharp.Test.ScriptHelpers
open FSharp.Test
open FSharp.Test.Utilities

open Internal.Utilities
Expand Down Expand Up @@ -148,8 +149,7 @@ type DependencyManagerInteractiveTests() =
Assert.Equal(0, result.Roots |> Seq.length)
()


[<Fact(Skip="failing on main")>]
[<Fact>]
member _.``Multiple Instances of DependencyProvider should be isolated``() =

let assemblyProbingPaths () = Seq.empty<string>
Expand Down Expand Up @@ -721,101 +721,65 @@ x |> Seq.iter(fun r ->

[<Fact>]
member _.``Verify that #help produces help text for fsi + dependency manager``() =
let expected = [|
""" F# Interactive directives:"""
""""""
""" #r "file.dll";; // Reference (dynamically load) the given DLL"""
""" #i "package source uri";; // Include package source uri when searching for packages"""
""" #I "path";; // Add the given search path for referenced DLLs"""
""" #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced"""
""" #time ["on"|"off"];; // Toggle timing on/off"""
""" #clear;; // Clear screen"""
""" #help;; // Display help"""
""" #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;"""
""" #quit;; // Exit"""
""""""
""" F# Interactive command line options:"""
""""""

// this is the end of the line each different platform has a different mechanism for starting fsi
// Actual output looks similar to: """ See 'testhost --help' for options"""
"""--help' for options"""

""""""
""""""
|]
let expected = """
F# Interactive directives:
#r "file.dll";; // Reference (dynamically load) the given DLL
#i "package source uri";; // Include package source uri when searching for packages
#I "path";; // Add the given search path for referenced DLLs
#load "file.fs" ...;; // Load the given file(s) as if compiled and referenced
#time ["on"|"off"];; // Toggle timing on/off
#help;; // Display help
#help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;
#clear;; // Clear screen
#quit;; // Exit
F# Interactive command line options:"""

let mutable found = 0
let lines = System.Collections.Generic.List()
use sawExpectedOutput = new ManualResetEvent(false)
let verifyOutput (line: string) =
let compareLine (s: string) =
if s = "" then line = ""
else line.EndsWith(s)
lines.Add(line)
match expected |> Array.tryFind(compareLine) with
| None -> ()
| Some t ->
found <- found + 1
if found = expected.Length then sawExpectedOutput.Set() |> ignore

let text = "#help"
use script = new FSharpScript(quiet = false, langVersion = LangVersion.V47)
let mutable found = 0
script.OutputProduced.Add (fun line -> verifyOutput line)
let opt = script.Eval(text) |> getValue
Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines)

use capture = new TestConsole.ExecutionCapture()
let opt = script.Eval("#help") |> getValue

let output = capture.OutText

Assert.Contains(expected, output)

// this is the end of the line each different platform has a different mechanism for starting fsi
// Actual output looks similar to: """ See 'testhost --help' for options"""
Assert.EndsWith("--help' for options", output.Trim())

[<Fact>]
member _.``Verify that #help produces help text for fsi + dependency manager language version preview``() =
let expected = [|
""" F# Interactive directives:"""
""""""
""" #r "file.dll";; // Reference (dynamically load) the given DLL"""
""" #i "package source uri";; // Include package source uri when searching for packages"""
""" #I "path";; // Add the given search path for referenced DLLs"""
""" #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced"""
""" #time ["on"|"off"];; // Toggle timing on/off"""
""" #help;; // Display help"""
""" #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;"""
""" #r "nuget:FSharp.Data, 3.1.2";; // Load Nuget Package 'FSharp.Data' version '3.1.2'"""
""" #r "nuget:FSharp.Data";; // Load Nuget Package 'FSharp.Data' with the highest version"""
""" #clear;; // Clear screen"""
""" #quit;; // Exit"""
""""""
""" F# Interactive command line options:"""
""""""

// this is the end of the line each different platform has a different mechanism for starting fsi
// Actual output looks similar to: """ See 'testhost --help' for options"""
"""--help' for options"""

""""""
""""""
|]
let expected = """
F# Interactive directives:
#r "file.dll";; // Reference (dynamically load) the given DLL
#i "package source uri";; // Include package source uri when searching for packages
#I "path";; // Add the given search path for referenced DLLs
#load "file.fs" ...;; // Load the given file(s) as if compiled and referenced
#time ["on"|"off"];; // Toggle timing on/off
#help;; // Display help
#help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;
#r "nuget:FSharp.Data, 3.1.2";; // Load Nuget Package 'FSharp.Data' version '3.1.2'
#r "nuget:FSharp.Data";; // Load Nuget Package 'FSharp.Data' with the highest version
#clear;; // Clear screen
#quit;; // Exit
F# Interactive command line options:"""

let mutable found = 0
let lines = System.Collections.Generic.List()
use sawExpectedOutput = new ManualResetEvent(false)
let verifyOutput (line: string) =
let compareLine (s: string) =
if s = "" then line = ""
else line.EndsWith(s)
lines.Add(line)
match expected |> Array.tryFind(compareLine) with
| None -> ()
| Some t ->
found <- found + 1
if found = expected.Length then sawExpectedOutput.Set() |> ignore

let text = "#help"
use script = new FSharpScript(quiet = false, langVersion = LangVersion.Preview)
let mutable found = 0
script.OutputProduced.Add (fun line -> verifyOutput line)
let opt = script.Eval(text) |> getValue
Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines)

use capture = new TestConsole.ExecutionCapture()
let opt = script.Eval("#help") |> getValue

let output = capture.OutText

Assert.Contains(expected, output)

// this is the end of the line each different platform has a different mechanism for starting fsi
// Actual output looks similar to: """ See 'testhost --help' for options"""
Assert.EndsWith("--help' for options", output.Trim())

[<Fact>]
member _.``Verify that timeout --- times out and fails``() =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
</PropertyGroup>

<ItemGroup>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="$(FSharpSourcesRoot)\Compiler\Interactive\ControlledExecution.fs" />
<Compile Include="$(FSharpSourcesRoot)\Compiler\Utilities\RidHelpers.fs" />
<Compile Include="DependencyManagerInteractiveTests.fs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace FSharp.Compiler.Scripting.UnitTests

open System
open System.Text
open System.Diagnostics
open System.IO
open System.Reflection
Expand All @@ -11,6 +12,7 @@ open System.Threading
open System.Threading.Tasks
open FSharp.Compiler.Interactive
open FSharp.Compiler.Interactive.Shell
open FSharp.Test
open FSharp.Test.ScriptHelpers

open Xunit
Expand Down Expand Up @@ -87,22 +89,20 @@ x

[<Fact>]
member _.``Capture console input``() =
use script = new FSharpScript(input = "stdin:1234\r\n")
use _ = new TestConsole.ProvideInput("stdin:1234\r\n")
use script = new FSharpScript()
let opt = script.Eval("System.Console.ReadLine()") |> getValue
let value = opt.Value
Assert.Equal(typeof<string>, value.ReflectionType)
Assert.Equal("stdin:1234", downcast value.ReflectionValue)

[<Fact>]
member _.``Capture console output/error``() =
use capture = new TestConsole.ExecutionCapture()
use script = new FSharpScript()
use sawOutputSentinel = new ManualResetEvent(false)
use sawErrorSentinel = new ManualResetEvent(false)
script.OutputProduced.Add (fun line -> if line = "stdout:1234" then sawOutputSentinel.Set() |> ignore)
script.ErrorProduced.Add (fun line -> if line = "stderr:5678" then sawErrorSentinel.Set() |> ignore)
script.Eval("printfn \"stdout:1234\"; eprintfn \"stderr:5678\"") |> ignoreValue
Assert.True(sawOutputSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see output sentinel value written")
Assert.True(sawErrorSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see error sentinel value written")
Assert.Contains("stdout:1234", capture.OutText)
Assert.Contains("stderr:5678", capture.ErrorText)

[<Fact>]
member _.``Maintain state between submissions``() =
Expand Down Expand Up @@ -306,30 +306,26 @@ printfn "{@"%A"}" result

[<Fact>]
member _.``Eval script with invalid PackageName should fail immediately``() =
use capture = new TestConsole.ExecutionCapture()
use script = new FSharpScript(additionalArgs=[| |])
let mutable found = 0
let outp = System.Collections.Generic.List<string>()
script.OutputProduced.Add(
fun line ->
if line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package") then
found <- found + 1
outp.Add(line))
let result, errors = script.Eval("""#r "nuget:FSharp.Really.Not.A.Package" """)
Assert.True( (found = 0), "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'")

let lines = capture.OutText.Split([| Environment.NewLine |], StringSplitOptions.None)
let found = lines |> Seq.exists (fun line -> line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package"))
Assert.False(found, "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'")
Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("error NU1101:")), "Expect to error containing 'error NU1101:'")
Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")), "Expect to error containing 'FSharp.Really.Not.A.Package'")

[<Fact>]
member _.``Eval script with invalid PackageName should fail immediately and resolve one time only``() =
use capture = new TestConsole.ExecutionCapture()
use script = new FSharpScript(additionalArgs=[| |])
let mutable foundResolve = 0
script.OutputProduced.Add (fun line -> if line.Contains("error NU1101:") then foundResolve <- foundResolve + 1)
let result, errors =
script.Eval("""
#r "nuget:FSharp.Really.Not.A.Package"
#r "nuget:FSharp.Really.Not.Another.Package"
""")
Assert.True( (foundResolve = 0), (sprintf "Did not expected to see 'error NU1101:' in output" ))
Assert.DoesNotContain("error NU1101:", capture.OutText)
Assert.Equal(2, (errors |> Seq.filter (fun error -> error.Message.Contains("error NU1101:")) |> Seq.length))
Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")) |> Seq.length))
Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.Another.Package")) |> Seq.length))
Expand Down Expand Up @@ -479,6 +475,7 @@ let x =
script.Eval(code) |> ignoreValue
Assert.False(foundInner)


[<Fact>]
member _.``Script with nuget package that yields out of order dependencies works correctly``() =
// regression test for: https://github.com/dotnet/fsharp/issues/9217
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "ifAvailable",
"shadowCopy": false,
"appDomain": "denied",
"parallelizeTestCollections": false,
"maxParallelThreads": 1
}
2 changes: 1 addition & 1 deletion tests/FSharp.Compiler.Service.Tests/Common.fs
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ let assertRange
[<AutoOpen>]
module TempDirUtils =
let getTempPath dir =
Path.Combine(TestFramework.tempDirectoryOfThisTestRun, dir)
Path.Combine(tempDirectoryOfThisTestRun.Value.FullName, dir)

/// Returns the file name part of a temp file name created with tryCreateTemporaryFileName ()
/// and an added process id and thread id to ensure uniqueness between threads.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
</Content>
<Compile Include="AssemblyInfo.fs" />
<Compile Include="SurfaceArea.fs" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="Common.fs" />
<Compile Include="GeneratedCodeSymbolsTests.fs" />
<Compile Include="AssemblyReaderShim.fs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4401,7 +4401,7 @@ let ``Test Project33 extension methods`` () =
("GetValue", ["member"; "extmem"])]

module internal Project34 =
let directoryPath = createTemporaryDirectory "Project34"
let directoryPath = createTemporaryDirectory().FullName
let sourceFileName = Path.Combine(directoryPath, "Program.fs")
let dllName = Path.ChangeExtension(sourceFileName, ".dll")
let projFileName = Path.ChangeExtension(sourceFileName, ".fsproj")
Expand Down
5 changes: 2 additions & 3 deletions tests/FSharp.Compiler.Service.Tests/xunit.runner.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "ifAvailable",
"shadowCopy": false
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "denied"
}
3 changes: 3 additions & 0 deletions tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
</PropertyGroup>

<ItemGroup>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="TestFrameworkHelpers.fs" />
<Compile Include="LibraryTestFx.fs" />
<Compile Include="FSharp.Core\PrimTypes.fs" />
Expand Down
9 changes: 4 additions & 5 deletions tests/FSharp.Core.UnitTests/xunit.runner.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "ifAvailable",
"shadowCopy": false,
"parallelizeTestCollections": false,
"maxParallelThreads": 1
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "denied",
"parallelizeTestCollections": false,
"maxParallelThreads": 1
}
Loading

0 comments on commit fb69e58

Please sign in to comment.