Refactor Native /Assembly resolution (#8606)

* Refactor native resolution

* Fcs -- compile

* typo

* buildy

* Linux

* Update src/fsharp/CompileOps.fs

Co-Authored-By: Phillip Carter <pcarter@fastmail.com>

* feedback

* temp

* c#

* fix delete and fsharpcore reference.

* improve surface area

Co-authored-by: Phillip Carter <pcarter@fastmail.com>

Author: KevinRansom

fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj

@@ -522,11 +522,23 @@
     <Compile Include="$(FSharpSourcesRoot)/fsharp/DotNetFrameworkDependencies.fs">
       <Link>Driver\DotNetFrameworkDependencies.fs</Link>
     </Compile>
-    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/DependencyManager.fsi">
-      <Link>Driver/DependencyManager.fsi</Link>
+    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/AssemblyResolveHandler.fsi">
+      <Link>Driver\AssemblyResolveHandler.fsi</Link>
     </Compile>
-    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/DependencyManager.fs">
-      <Link>Driver/DependencyManager.fs</Link>
+    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/AssemblyResolveHandler.fs">
+      <Link>Driver\AssemblyResolveHandler.fs</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/NativeDllResolveHandler.fsi">
+      <Link>Driver\NativeDllResolveHandler.fsi</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/NativeDllResolveHandler.fs">
+      <Link>Driver\NativeDllResolveHandler.fs</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/DependencyProvider.fsi">
+      <Link>Driver/DependencyProvider.fsi</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)/fsharp/Interactive.DependencyManager/DependencyProvider.fs">
+      <Link>Driver/DependencyProvider.fs</Link>
     </Compile>
     <Compile Include="$(FSharpSourcesRoot)/fsharp/CompileOps.fsi">
       <Link>Driver/CompileOps.fsi</Link>

src/fsharp/CompileOps.fs

@@ -2199,7 +2199,7 @@ type TcConfigBuilder =
       mutable productNameForBannerText: string
       /// show the MS (c) notice, e.g. with help or fsi? 
       mutable showBanner: bool
-        
+
       /// show times between passes? 
       mutable showTimes: bool
       mutable showLoadedAssemblies: bool
@@ -2229,7 +2229,7 @@ type TcConfigBuilder =
       mutable emitDebugInfoInQuotations: bool
 
       mutable exename: string option
-      
+
       // If true - the compiler will copy FSharp.Core.dll along the produced binaries
       mutable copyFSharpCore: CopyFSharpCoreFlag
 
@@ -2390,9 +2390,24 @@ type TcConfigBuilder =
           noConditionalErasure = false
           pathMap = PathMap.empty
           langVersion = LanguageVersion("default")
-          dependencyProvider = new DependencyProvider()
+          dependencyProvider = Unchecked.defaultof<DependencyProvider>
         }
 
+    // Directories to start probing in
+    // Algorithm:
+    //  Search for native libraries using:
+    //  1. Include directories
+    //  2. compilerToolPath directories
+    //  3. reference dll's
+    //  4. The implicit include directory
+    member private tcConfigB.nativeProbingRoots () =
+        seq {
+            yield! tcConfigB.includes
+            yield! tcConfigB.compilerToolPaths
+            yield! (tcConfigB.referencedDLLs |> Seq.map(fun ref -> Path.GetDirectoryName(ref.Text)))
+            yield tcConfigB.implicitIncludeDir
+        } |> Seq.distinct
+
     static member CreateNew(legacyReferenceResolver, defaultFSharpBinariesDir, reduceMemoryUsage, implicitIncludeDir,
                             isInteractive, isInvalidationSupported, defaultCopyFSharpCore, tryGetMetadataSnapshot) =
 
@@ -2401,17 +2416,20 @@ type TcConfigBuilder =
         if (String.IsNullOrEmpty defaultFSharpBinariesDir) then
             failwith "Expected a valid defaultFSharpBinariesDir"
 
-        { TcConfigBuilder.Initial with 
-            implicitIncludeDir = implicitIncludeDir
-            defaultFSharpBinariesDir = defaultFSharpBinariesDir
-            reduceMemoryUsage = reduceMemoryUsage
-            legacyReferenceResolver = legacyReferenceResolver
-            isInteractive = isInteractive
-            isInvalidationSupported = isInvalidationSupported
-            copyFSharpCore = defaultCopyFSharpCore
-            tryGetMetadataSnapshot = tryGetMetadataSnapshot
-            useFsiAuxLib = isInteractive
-        }
+        let tcConfigBuilder =
+            { TcConfigBuilder.Initial with 
+                implicitIncludeDir = implicitIncludeDir
+                defaultFSharpBinariesDir = defaultFSharpBinariesDir
+                reduceMemoryUsage = reduceMemoryUsage
+                legacyReferenceResolver = legacyReferenceResolver
+                isInteractive = isInteractive
+                isInvalidationSupported = isInvalidationSupported
+                copyFSharpCore = defaultCopyFSharpCore
+                tryGetMetadataSnapshot = tryGetMetadataSnapshot
+                useFsiAuxLib = isInteractive
+            }
+        tcConfigBuilder.dependencyProvider <- new DependencyProvider(NativeResolutionProbe(tcConfigBuilder.nativeProbingRoots))
+        tcConfigBuilder
 
     member tcConfigB.ResolveSourceFile(m, nm, pathLoadedFrom) = 
         use unwindBuildPhase = PushThreadBuildPhaseUntilUnwind BuildPhase.Parameter
@@ -4964,10 +4982,13 @@ let ProcessMetaCommandsFromInput
                 if not canHaveScriptMetaCommands then
                     errorR(HashReferenceNotAllowedInNonScript m)
 
-                let reportError errorType error =
-                    match errorType with
-                    | ErrorReportType.Warning -> warning(Error(error,m))
-                    | ErrorReportType.Error -> errorR(Error(error, m))
+                let reportError =
+                    let report errorType err msg =
+                        let error = err, msg
+                        match errorType with
+                        | ErrorReportType.Warning -> warning(Error(error, m))
+                        | ErrorReportType.Error -> errorR(Error(error, m))
+                    ResolvingErrorReport (report)
 
                 match args with
                 | [path] ->
@@ -5257,10 +5278,13 @@ module ScriptPreprocessClosure =
                     match packageManagerLines with
                     | [] -> ()
                     | (_, _, m)::_ ->
-                        let reportError errorType error =
-                             match errorType with
-                             | ErrorReportType.Warning -> warning(Error(error,m))
-                             | ErrorReportType.Error -> errorR(Error(error, m))
+                        let reportError =
+                            let report errorType err msg =
+                                let error = err, msg
+                                match errorType with
+                                | ErrorReportType.Warning -> warning(Error(error, m))
+                                | ErrorReportType.Error -> errorR(Error(error, m))
+                            ResolvingErrorReport (report)
 
                         match origTcConfig.packageManagerLines |> Map.tryFind packageManagerKey with
                         | Some oldDependencyManagerLines when oldDependencyManagerLines = packageManagerLines -> ()
@@ -5273,22 +5297,23 @@ module ScriptPreprocessClosure =
                             | dependencyManager ->
                                 let inline snd3 (_, b, _) = b
                                 let packageManagerTextLines = packageManagerLines |> List.map snd3
-                                match tcConfig.dependencyProvider.Resolve(dependencyManager, tcConfig.implicitIncludeDir, mainFile, scriptName, ".fsx", packageManagerTextLines, reportError, executionTfm) with
-                                | true, _references, generatedScripts, additionalIncludeFolders ->
+                                let result = tcConfig.dependencyProvider.Resolve(dependencyManager, ".fsx", packageManagerTextLines, reportError, executionTfm, tcConfig.implicitIncludeDir, mainFile, scriptName)
+                                match result.Success with
+                                | true ->
                                     // Resolution produced no errors
-                                    if not (Seq.isEmpty additionalIncludeFolders) then
+                                    if not (Seq.isEmpty result.Roots) then
                                         let tcConfigB = tcConfig.CloneOfOriginalBuilder
-                                        for folder in additionalIncludeFolders do 
+                                        for folder in result.Roots do 
                                             tcConfigB.AddIncludePath(m, folder, "")
                                         tcConfigB.packageManagerLines <- tcConfigB.packageManagerLines |> Map.map(fun _ l -> l |> List.map(fun (_, p, m) -> true, p, m))
                                         tcConfig <- TcConfig.Create(tcConfigB, validate=false)
-                                    for script in generatedScripts do
+                                    for script in result.SourceFiles do
                                         let scriptText = File.ReadAllText script
                                         loadScripts.Add script |> ignore
                                         let iSourceText = SourceText.ofString scriptText
                                         yield! loop (ClosureSource(script, m, iSourceText, true))
 
-                                | false, _, _, _ ->
+                                | false ->
                                     // Resolution produced errors update packagerManagerLines entries to note these failure
                                     // failed resolutions will no longer be considered
                                     let tcConfigB = tcConfig.CloneOfOriginalBuilder

src/fsharp/FSharp.DependencyManager.Nuget/FSDependencyManager.txt

@@ -1,2 +1,5 @@
 # FSharp.Build resource strings 
+cantReferenceSystemPackage,"PackageManager can not reference the System Package '%s'"
+requiresAValue,"%s requires a value"
+unableToApplyImplicitArgument,"Unable to apply implicit argument number %d"
 notUsed,"Not used."

src/fsharp/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.ProjectFile.fs

@@ -187,7 +187,7 @@ $(PACKAGEREFERENCES)
     <ItemGroup>
       <ResolvedReferenceLines Remove='*' />
       <ResolvedReferenceLines
-          Condition="'$(SCRIPTEXTENSION)'=='.fsx' and '%(InteractiveResolvedFile.NugetPackageId)'!='FSharp.Core'"
+          Condition="'$(SCRIPTEXTENSION)'=='.csx' or '%(InteractiveResolvedFile.NugetPackageId)'!='FSharp.Core'"
           Include='%(InteractiveResolvedFile.NugetPackageId),%(InteractiveResolvedFile.NugetPackageVersion),%(InteractiveResolvedFile.PackageRoot),%(InteractiveResolvedFile.FullPath),%(InteractiveResolvedFile.IsNotImplementationReference),%(InteractiveResolvedFile.InitializeSourcePath),%(NativeIncludeRoots.Path)'
           KeepDuplicates="false" />
     </ItemGroup>

src/fsharp/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.Utilities.fs

@@ -152,16 +152,16 @@ module internal Utilities =
         else
             None
 
-    let drainStreamToFile (stream: StreamReader) filename =
-        use file = File.OpenWrite(filename)
-        use writer = new StreamWriter(file)
+    let drainStreamToMemory (stream: StreamReader) =
+        let mutable list = ResizeArray()
         let rec copyLines () =
             match stream.ReadLine() with
             | null -> ()
             | line ->
-                writer.WriteLine(line)
+                list.Add line
                 copyLines ()
         copyLines ()
+        list.ToArray()
 
     let executeBuild pathToExe arguments workingDir =
         match pathToExe with
@@ -179,24 +179,24 @@ module internal Utilities =
             p.StartInfo <- psi
             p.Start() |> ignore
 
-            let standardOutput = Path.Combine(workingDir, "StandardOutput.txt")
-            let standardError = Path.Combine(workingDir, "StandardError.txt")
-            drainStreamToFile p.StandardOutput (Path.Combine(workingDir, standardOutput))
-            drainStreamToFile p.StandardError (Path.Combine(workingDir, standardError))
+            let stdOut = drainStreamToMemory p.StandardOutput
+            let stdErr = drainStreamToMemory p.StandardError
+
+#if Debug
+            File.WriteAllLines(Path.Combine(workingDir, "StandardOutput.txt"), stdOut)
+            File.WriteAllLines(Path.Combine(workingDir, "StandardError.txt"), stdErr)
+#endif
 
             p.WaitForExit()
+
             if p.ExitCode <> 0 then
                 //Write StandardError.txt to err stream
-                let text = File.ReadAllText(standardOutput)
-                Console.Out.Write(text)
-
-                //Write StandardOutput.txt to out stream
-                let text = File.ReadAllText(standardError)
-                Console.Out.Write(text)
+                for line in stdOut do Console.Out.WriteLine(line)
+                for line in stdErr do Console.Error.WriteLine(line)
 
-            p.ExitCode = 0
+            p.ExitCode = 0, stdOut, stdErr
 
-        | None -> false
+        | None -> false, Array.empty, Array.empty
 
     let buildProject projectPath binLogPath =
         let binLoggingArguments =
@@ -213,7 +213,7 @@ module internal Utilities =
 
         let workingDir = Path.GetDirectoryName projectPath
 
-        let succeeded =
+        let succeeded, stdOut, stdErr =
             if not (isRunningOnCoreClr) then
                 // The Desktop build uses "msbuild" to build
                 executeBuild msbuildExePath (arguments "") workingDir
@@ -223,4 +223,4 @@ module internal Utilities =
 
         let outputFile = projectPath + ".resolvedReferences.paths"
         let resultOutFile = if succeeded && File.Exists(outputFile) then Some outputFile else None
-        succeeded, resultOutFile
+        succeeded, stdOut, stdErr, resultOutFile

src/fsharp/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.fs

@@ -10,7 +10,7 @@ open System.IO
 open FSharp.DependencyManager.Nuget
 open FSharp.DependencyManager.Nuget.Utilities
 open FSharp.DependencyManager.Nuget.ProjectFile
-
+open FSDependencyManager
 
 module FSharpDependencyManager =
 
@@ -43,7 +43,7 @@ module FSharpDependencyManager =
         let parsePackageReferenceOption (line: string) =
             let validatePackageName package packageName =
                 if String.Compare(packageName, package, StringComparison.OrdinalIgnoreCase) = 0 then
-                    raise (ArgumentException(sprintf "PackageManager can not reference the System Package '%s'" packageName))  // @@@@@@@@@@@@@@@@@@@@@@@ Globalize me please
+                    raise (ArgumentException(SR.cantReferenceSystemPackage(packageName)))
             let rec parsePackageReferenceOption' (options: (string option * string option) list) (implicitArgumentCount: int) (packageReference: PackageReference option) =
                 let current =
                     match packageReference with
@@ -61,11 +61,11 @@ module FSharpDependencyManager =
                     let setVersion v = Some { current with Version = v }
                     match opt with
                     | Some "include", Some v -> addInclude v |> parsePackageReferenceOption' rest implicitArgumentCount
-                    | Some "include", None -> raise (ArgumentException(sprintf "%s requires a value" "Include"))               // @@@@@@@@@@@@@@@@@@@@@@@ Globalize me please
+                    | Some "include", None -> raise (ArgumentException(SR.requiresAValue("Include")))
                     | Some "version", Some v -> setVersion v |> parsePackageReferenceOption' rest implicitArgumentCount
-                    | Some "version", None -> raise (ArgumentException(sprintf "%s requires a value" "Version"))               // @@@@@@@@@@@@@@@@@@@@@@@ Globalize me please
+                    | Some "version", None -> setVersion "*" |> parsePackageReferenceOption' rest implicitArgumentCount
                     | Some "restoresources", Some v -> Some { current with RestoreSources = concat current.RestoreSources v } |> parsePackageReferenceOption' rest implicitArgumentCount
-                    | Some "restoresources", None -> raise (ArgumentException(sprintf "%s requires a value" "RestoreSources")) // @@@@@@@@@@@@@@@@@@@@@@@ Globalize me please
+                    | Some "restoresources", None -> raise (ArgumentException(SR.requiresAValue("RestoreSources")))
                     | Some "script", Some v -> Some { current with Script = v } |> parsePackageReferenceOption' rest implicitArgumentCount
                     | Some "bl", value ->
                         match value with
@@ -88,7 +88,7 @@ module FSharpDependencyManager =
                             match implicitArgumentCount with
                             | 0 -> addInclude v
                             | 1 -> setVersion v
-                            | _ -> raise (ArgumentException(sprintf "Unable to apply implicit argument number %d" (implicitArgumentCount + 1))) // @@@@@@@@@@@@@@@@@@@@@@@ Globalize me please
+                            | _ -> raise (ArgumentException(SR.unableToApplyImplicitArgument(implicitArgumentCount + 1)))
                             |> parsePackageReferenceOption' rest (implicitArgumentCount + 1)
                     | _ -> parsePackageReferenceOption' rest implicitArgumentCount packageReference
             let options = getOptions line
@@ -98,12 +98,34 @@ module FSharpDependencyManager =
         |> List.distinct
         |> (fun l -> l, binLogPath)
 
+
+/// The results of ResolveDependencies
+type ResolveDependenciesResult (success: bool, stdOut: string array, stdError: string array, resolutions: string seq, sourceFiles: string seq, roots: string seq) =
+
+    /// Succeded?
+    member __.Success = success
+
+    /// The resolution output log
+    member __.StdOut = stdOut
+
+    /// The resolution error log (* process stderror *)
+    member __.StdError = stdError
+
+    /// The resolution paths
+    member __.Resolutions = resolutions
+
+    /// The source code file paths
+    member __.SourceFiles = sourceFiles
+
+    /// The roots to package directories
+    member __.Roots = roots
+
 type [<DependencyManagerAttribute>] FSharpDependencyManager (outputDir:string option) =
 
     let key = "nuget"
     let name = "MsBuild Nuget DependencyManager"
     let scriptsPath =
-        let path = Path.Combine(Path.GetTempPath(), key, Process.GetCurrentProcess().Id.ToString())
+        let path = Path.Combine(Path.GetTempPath(), key, Process.GetCurrentProcess().Id.ToString() + "--"+ Guid.NewGuid().ToString())
         match outputDir with
         | None -> path
         | Some v -> Path.Combine(path, v)
@@ -112,8 +134,12 @@ type [<DependencyManagerAttribute>] FSharpDependencyManager (outputDir:string op
 
     let deleteScripts () =
         try
+#if !Debug
             if Directory.Exists(scriptsPath) then
-                () //Directory.Delete(scriptsPath, true)
+                Directory.Delete(scriptsPath, true)
+#else
+            ()
+#endif
         with | _ -> ()
 
     let deleteAtExit =
@@ -136,7 +162,7 @@ type [<DependencyManagerAttribute>] FSharpDependencyManager (outputDir:string op
 
     member __.Key = key
 
-    member __.ResolveDependencies(scriptExt:string, packageManagerTextLines:string seq, tfm: string) : bool * string seq * string seq * string seq =
+    member __.ResolveDependencies(scriptExt:string, packageManagerTextLines:string seq, tfm: string) : obj =
 
         let scriptExt, poundRprefix  =
             match scriptExt with
@@ -170,7 +196,7 @@ type [<DependencyManagerAttribute>] FSharpDependencyManager (outputDir:string op
 
             writeFile projectPath generateProjBody
 
-            let result, resolutionsFile = buildProject projectPath binLogPath
+            let result, stdOut, stdErr,  resolutionsFile = buildProject projectPath binLogPath
             match resolutionsFile with
             | Some file ->
                 let resolutions = getResolutionsFromFile file
@@ -183,10 +209,10 @@ type [<DependencyManagerAttribute>] FSharpDependencyManager (outputDir:string op
                     List.concat [ [scriptPath]; loads] |> List.toSeq
                 let includes = (findIncludesFromResolutions resolutions) |> Array.toSeq
 
-                result, references, scripts, includes
+                ResolveDependenciesResult(result, stdOut, stdErr, references, scripts, includes)
 
             | None ->
                 let empty = Seq.empty<string>
-                result, empty, empty, empty
+                ResolveDependenciesResult(result, stdOut, stdErr, empty, empty, empty)
 
-        generateAndBuildProjectArtifacts
+        generateAndBuildProjectArtifacts :> obj