Stack overflow fixes (#7151)

* Trying to fix the stack overflow case in ilxgen

* Added stack guard to GenExpr

* Still trying to resolve this

* This resolves the stack overflow, but need to review

* Adding better tests

* Fixing compile and run successfully assert

* One test passes

* Fixing a few things

* Delete fieldNames.txt

* Update IlxGen.fs

* Update IlxGen.fs

* Update IlxGen.fs

* More work on CompilerAssert

* Got verifying IL working on method

* Always delay gen methods when already gen'ing in one. Some small refactoring, created IlxMethodInfo type instead of using large tuples.

* Better error message for stack overflow

* small msg change

* Simplifying some tests

* Removing attempt at bringing a FSharpQA test to nunit

* Fixing up seq points

* More seq point tweaks

* Trying to fix ilxgen

* Fixing ilxgen

* Using RunScript

* Fix test

* removed IlxMethodInfo

* trivial refactor

* another quick refactor

src/fsharp/lib.fs

@@ -536,3 +536,14 @@ module UnmanagedProcessExecutionOptions =
                             "HeapSetInformation() returned FALSE; LastError = 0x" + 
                             GetLastError().ToString("X").PadLeft(8, '0') + "."))
 
+[<RequireQualifiedAccess>]
+module StackGuard =
+
+    open System.Runtime.CompilerServices
+
+    [<Literal>] 
+    let private MaxUncheckedRecursionDepth = 20
+
+    let EnsureSufficientExecutionStack recursionDepth =
+        if recursionDepth > MaxUncheckedRecursionDepth then
+            RuntimeHelpers.EnsureSufficientExecutionStack ()

tests/fsharp/Compiler/CompilerAssert.fs

@@ -11,6 +11,19 @@ open FSharp.Compiler.SourceCodeServices
 open FSharp.Compiler.Interactive.Shell
 
 open NUnit.Framework
+open System.Reflection.Emit
+
+[<Sealed>]
+type ILVerifier (dllFilePath: string) =
+
+    member this.VerifyIL (qualifiedItemName: string, expectedIL: string) =
+        ILChecker.checkILItem qualifiedItemName dllFilePath [ expectedIL ]
+
+    member this.VerifyIL (expectedIL: string list) =
+        ILChecker.checkIL dllFilePath expectedIL
+
+    member this.VerifyILWithLineNumbers (qualifiedItemName: string, expectedIL: string) =
+        ILChecker.checkILItemWithLineNumbers qualifiedItemName dllFilePath [ expectedIL ]
 
 [<RequireQualifiedAccess>]
 module CompilerAssert =
@@ -48,10 +61,45 @@ module CompilerAssert =
             Stamp = None
         }
 
-    let lockObj = obj ()
+    let private gate = obj ()
+
+    let private compile isExe source f =
+        lock gate <| fun () ->
+            let inputFilePath = Path.ChangeExtension(Path.GetTempFileName(), ".fs")
+            let outputFilePath = Path.ChangeExtension (Path.GetTempFileName(), if isExe then ".exe" else ".dll")
+            let runtimeConfigFilePath = Path.ChangeExtension (outputFilePath, ".runtimeconfig.json")
+            let fsCoreDllPath = config.FSCOREDLLPATH
+            let tmpFsCoreFilePath = Path.Combine (Path.GetDirectoryName(outputFilePath), Path.GetFileName(fsCoreDllPath))
+            try
+                File.Copy (fsCoreDllPath , tmpFsCoreFilePath, true)
+                File.WriteAllText (inputFilePath, source)
+                File.WriteAllText (runtimeConfigFilePath, """
+{
+  "runtimeOptions": {
+    "tfm": "netcoreapp2.1",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "2.1.0"
+    }
+  }
+}
+                """)
+
+                let args =
+                    defaultProjectOptions.OtherOptions
+                    |> Array.append [| "fsc.exe"; inputFilePath; "-o:" + outputFilePath; (if isExe then "--target:exe" else "--target:library"); "--nowin32manifest" |]
+                let errors, _ = checker.Compile args |> Async.RunSynchronously
+
+                f (errors, outputFilePath)
+
+            finally
+                try File.Delete inputFilePath with | _ -> ()
+                try File.Delete outputFilePath with | _ -> ()
+                try File.Delete runtimeConfigFilePath with | _ -> ()
+                try File.Delete tmpFsCoreFilePath with | _ -> ()
 
     let Pass (source: string) =
-        lock lockObj <| fun () ->
+        lock gate <| fun () ->
             let parseResults, fileAnswer = checker.ParseAndCheckFileInProject("test.fs", 0, SourceText.ofString source, defaultProjectOptions) |> Async.RunSynchronously
 
             Assert.IsEmpty(parseResults.Errors, sprintf "Parse errors: %A" parseResults.Errors)
@@ -64,7 +112,7 @@ module CompilerAssert =
 
 
     let TypeCheckSingleError (source: string) (expectedErrorNumber: int) (expectedErrorRange: int * int * int * int) (expectedErrorMsg: string) =
-        lock lockObj <| fun () ->
+        lock gate <| fun () ->
             let parseResults, fileAnswer = checker.ParseAndCheckFileInProject("test.fs", 0, SourceText.ofString source, defaultProjectOptions) |> Async.RunSynchronously
 
             Assert.IsEmpty(parseResults.Errors, sprintf "Parse errors: %A" parseResults.Errors)
@@ -82,8 +130,46 @@ module CompilerAssert =
                 Assert.AreEqual(expectedErrorMsg, info.Message, "expectedErrorMsg")
             )
 
+    let CompileExe (source: string) =
+        compile true source (fun (errors, _) -> 
+            if errors.Length > 0 then
+                Assert.Fail (sprintf "Compile had warnings and/or errors: %A" errors))
+
+    let CompileExeAndRun (source: string) =
+        compile true source (fun (errors, outputExe) ->
+
+            if errors.Length > 0 then
+                Assert.Fail (sprintf "Compile had warnings and/or errors: %A" errors)
+
+            let pInfo = ProcessStartInfo ()
+#if NETCOREAPP
+            pInfo.FileName <- config.DotNetExe
+            pInfo.Arguments <- outputExe
+#else
+            pInfo.FileName <- outputExe
+#endif
+
+            pInfo.RedirectStandardError <- true
+            pInfo.UseShellExecute <- false
+
+            let p = Process.Start(pInfo)
+
+            p.WaitForExit()
+            let errors = p.StandardError.ReadToEnd ()
+            if not (String.IsNullOrWhiteSpace errors) then
+                Assert.Fail errors
+        )
+
+    let CompileLibraryAndVerifyIL (source: string) (f: ILVerifier -> unit) =
+        compile false source (fun (errors, outputFilePath) -> 
+            if errors.Length > 0 then
+                Assert.Fail (sprintf "Compile had warnings and/or errors: %A" errors)
+
+            f (ILVerifier outputFilePath)
+        )
+
     let RunScript (source: string) (expectedErrorMessages: string list) =
-        lock lockObj <| fun () ->
+        lock gate <| fun () ->
             // Intialize output and input streams
             use inStream = new StringReader("")
             use outStream = new StringWriter()

tests/fsharp/Compiler/ILChecker.fs

@@ -0,0 +1,97 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
+
+namespace FSharp.Compiler.UnitTests
+
+open System
+open System.IO
+open System.Diagnostics
+
+open NUnit.Framework
+open TestFramework
+
+[<RequireQualifiedAccess>]
+module ILChecker =
+
+    let config = initializeSuite ()
+
+    let private exec exe args =
+        let startInfo = ProcessStartInfo(exe, String.concat " " args)
+        startInfo.RedirectStandardError <- true
+        startInfo.UseShellExecute <- false
+        use p = Process.Start(startInfo)
+        p.WaitForExit()
+        p.StandardError.ReadToEnd(), p.ExitCode
+
+    /// Filters i.e ['The system type \'System.ReadOnlySpan`1\' was required but no referenced system DLL contained this type']
+    let private filterSpecialComment (text: string) =
+        let pattern = @"(\[\'(.*?)\'\])"
+        System.Text.RegularExpressions.Regex.Replace(text, pattern,
+            (fun me -> String.Empty)
+        )
+
+    let private checkILAux ildasmArgs dllFilePath expectedIL =
+        let ilFilePath = Path.ChangeExtension(dllFilePath, ".il")
+
+        let mutable errorMsgOpt = None
+        try
+            let ildasmPath = config.ILDASM
+
+            exec ildasmPath (ildasmArgs @ [ sprintf "%s /out=%s" dllFilePath ilFilePath ]) |> ignore
+
+            let text = File.ReadAllText(ilFilePath)
+            let blockComments = @"/\*(.*?)\*/"
+            let lineComments = @"//(.*?)\r?\n"
+            let strings = @"""((\\[^\n]|[^""\n])*)"""
+            let verbatimStrings = @"@(""[^""]*"")+"
+            let textNoComments =
+                System.Text.RegularExpressions.Regex.Replace(text, 
+                    blockComments + "|" + lineComments + "|" + strings + "|" + verbatimStrings, 
+                    (fun me -> 
+                        if (me.Value.StartsWith("/*") || me.Value.StartsWith("//")) then
+                            if me.Value.StartsWith("//") then Environment.NewLine else String.Empty
+                        else
+                            me.Value), System.Text.RegularExpressions.RegexOptions.Singleline)
+                |> filterSpecialComment
+
+            expectedIL
+            |> List.iter (fun (ilCode: string) ->
+                let expectedLines = ilCode.Split('\n')
+                let startIndex = textNoComments.IndexOf(expectedLines.[0])
+                if startIndex = -1 || textNoComments.Length < startIndex + ilCode.Length then
+                    errorMsgOpt <- Some("==EXPECTED CONTAINS==\n" + ilCode + "\n")
+                else
+                    let errors = ResizeArray()
+                    let actualLines = textNoComments.Substring(startIndex, textNoComments.Length - startIndex).Split('\n')
+                    for i = 0 to expectedLines.Length - 1 do
+                        let expected = expectedLines.[i].Trim()
+                        let actual = actualLines.[i].Trim()
+                        if expected <> actual then
+                            errors.Add(sprintf "\n==\nName: %s\n\nExpected:\t %s\nActual:\t\t %s\n==" actualLines.[0] expected actual)
+
+                    if errors.Count > 0 then
+                        let msg = String.concat "\n" errors + "\n\n\n==EXPECTED==\n" + ilCode + "\n"
+                        errorMsgOpt <- Some(msg + "\n\n\n==ACTUAL==\n" + String.Join("\n", actualLines, 0, expectedLines.Length))
+            )
+
+            if expectedIL.Length = 0 then
+                errorMsgOpt <- Some ("No Expected IL")
+
+            match errorMsgOpt with
+            | Some(msg) -> errorMsgOpt <- Some(msg + "\n\n\n==ENTIRE ACTUAL==\n" + textNoComments)
+            | _ -> ()
+        finally
+            try File.Delete(ilFilePath) with | _ -> ()
+
+            match errorMsgOpt with
+            | Some(errorMsg) -> 
+                Assert.Fail(errorMsg)
+            | _ -> ()
+
+    let checkILItem item dllFilePath expectedIL =
+        checkILAux [ sprintf "/item:%s" item ] dllFilePath expectedIL
+
+    let checkILItemWithLineNumbers item dllFilePath expectedIL =
+        checkILAux [ sprintf "/item:\"%s\"" item; "/linenum" ] dllFilePath expectedIL
+
+    let checkIL dllFilePath expectedIL =
+        checkILAux [] dllFilePath expectedIL