[Experimental] [WIP] Transparent Compiler

src/Compiler/Driver/GraphChecking/Graph.fs

@@ -27,26 +27,42 @@ module internal Graph =
         |> Array.map (fun (KeyValue (k, v)) -> k, v)
         |> readOnlyDict
 
-    let transitive<'Node when 'Node: equality> (graph: Graph<'Node>) : Graph<'Node> =
-        /// Find transitive dependencies of a single node.
-        let transitiveDeps (node: 'Node) =
-            let visited = HashSet<'Node>()
+    let nodes (graph: Graph<'Node>) : Set<'Node> =
+        graph.Values |> Seq.collect id |> Seq.append graph.Keys |> Seq.distinct |> Set
+
+    /// Find transitive dependencies of a single node.
+    let transitiveDeps (node: 'Node) (graph: Graph<'Node>) =
+        let visited = HashSet<'Node>()
 
-            let rec dfs (node: 'Node) =
-                graph[node]
-                // Add direct dependencies.
-                // Use HashSet.Add return value semantics to filter out those that were added previously.
-                |> Array.filter visited.Add
-                |> Array.iter dfs
+        let rec dfs (node: 'Node) =
+            graph[node]
+            // Add direct dependencies.
+            // Use HashSet.Add return value semantics to filter out those that were added previously.
+            |> Array.filter visited.Add
+            |> Array.iter dfs
 
-            dfs node
-            visited |> Seq.toArray
+        dfs node
+        visited |> Seq.toArray
 
+    let transitive<'Node when 'Node: equality> (graph: Graph<'Node>) : Graph<'Node> =
         graph.Keys
         |> Seq.toArray
-        |> Array.Parallel.map (fun node -> node, transitiveDeps node)
+        |> Array.Parallel.map (fun node -> node, graph |> transitiveDeps node)
         |> readOnlyDict
 
+    /// Get subgraph of the given graph that contains only nodes that are reachable from the given node.
+    let subGraphFor node graph =
+        let allDeps = graph |> transitiveDeps node
+        let relevant n = n = node || allDeps |> Array.contains n
+
+        graph
+        |> Seq.choose (fun (KeyValue (src, deps)) ->
+            if relevant src then
+                Some(src, deps |> Array.filter relevant)
+            else
+                None)
+        |> make
+
     /// Create a reverse of the graph
     let reverse (originalGraph: Graph<'Node>) : Graph<'Node> =
         originalGraph
@@ -59,6 +75,48 @@ module internal Graph =
         |> readOnlyDict
         |> addIfMissing originalGraph.Keys
 
+    /// Returns leaves of the graph and the remaining graph without the leaves.
+    let cutLeaves (graph: Graph<'Node>) =
+        let notLeaves =
+            set
+                [
+                    for (KeyValue (node, deps)) in graph do
+                        if deps.Length > 0 then
+                            node
+                ]
+
+        let leaves =
+            set
+                [
+                    for (KeyValue (node, deps)) in graph do
+                        if deps.Length = 0 then
+                            node
+
+                        yield! deps |> Array.filter (notLeaves.Contains >> not)
+                ]
+
+        leaves,
+        seq {
+            for (KeyValue (node, deps)) in graph do
+                if deps.Length > 0 then
+                    node, deps |> Array.filter (leaves.Contains >> not)
+        }
+        |> make
+
+    /// Returns layers of leaves repeatedly removed from the graph until there's nothing left
+    let leafSequence (graph: Graph<'Node>) =
+        let rec loop (graph: Graph<'Node>) acc =
+            match graph |> cutLeaves with
+            | leaves, _ when leaves.IsEmpty -> acc
+            | leaves, graph ->
+                seq {
+                    yield! acc
+                    leaves
+                }
+                |> loop graph
+
+        loop graph Seq.empty
+
     let printCustom (graph: Graph<'Node>) (nodePrinter: 'Node -> string) : unit =
         printfn "Graph:"
         let join (xs: string[]) = System.String.Join(", ", xs)
@@ -69,7 +127,7 @@ module internal Graph =
     let print (graph: Graph<'Node>) : unit =
         printCustom graph (fun node -> node.ToString())
 
-    let serialiseToMermaid path (graph: Graph<FileIndex * string>) =
+    let serialiseToMermaid (graph: Graph<FileIndex * string>) =
         let sb = StringBuilder()
         let appendLine (line: string) = sb.AppendLine(line) |> ignore
 
@@ -84,8 +142,10 @@ module internal Graph =
                 appendLine $"    %i{idx} --> %i{depIdx}"
 
         appendLine "```"
+        sb.ToString()
 
+    let writeMermaidToFile path (graph: Graph<FileIndex * string>) =
         use out =
             FileSystem.OpenFileForWriteShim(path, fileMode = System.IO.FileMode.Create)
 
-        out.WriteAllText(sb.ToString())
+        graph |> serialiseToMermaid |> out.WriteAllText

src/Compiler/Driver/GraphChecking/Graph.fsi

@@ -10,12 +10,20 @@ module internal Graph =
     /// Build the graph.
     val make: nodeDeps: seq<'Node * 'Node array> -> Graph<'Node> when 'Node: equality
     val map<'T, 'U when 'U: equality> : f: ('T -> 'U) -> graph: Graph<'T> -> Graph<'U>
+    /// Get all nodes of the graph.
+    val nodes: graph: Graph<'Node> -> Set<'Node>
     /// Create a transitive closure of the graph in O(n^2) time (but parallelize it).
     /// The resulting graph contains edge A -> C iff the input graph contains a (directed) non-zero length path from A to C.
     val transitive<'Node when 'Node: equality> : graph: Graph<'Node> -> Graph<'Node>
+    /// Get a sub-graph of the graph containing only the nodes reachable from the given node.
+    val subGraphFor: node: 'Node -> graph: Graph<'Node> -> Graph<'Node> when 'Node: equality
     /// Create a reverse of the graph.
     val reverse<'Node when 'Node: equality> : originalGraph: Graph<'Node> -> Graph<'Node>
+    /// Returns layers of leaves repeatedly removed from the graph until there's nothing left
+    val leafSequence: graph: Graph<'Node> -> Set<'Node> seq
     /// Print the contents of the graph to the standard output.
     val print: graph: Graph<'Node> -> unit
+    /// Create a simple Mermaid graph
+    val serialiseToMermaid: graph: Graph<FileIndex * string> -> string
     /// Create a simple Mermaid graph and save it under the path specified.
-    val serialiseToMermaid: path: string -> graph: Graph<FileIndex * string> -> unit
+    val writeMermaidToFile: path: string -> graph: Graph<FileIndex * string> -> unit

src/Compiler/Driver/ParseAndCheckInputs.fs

@@ -1467,14 +1467,14 @@ let CheckOneInputWithCallback
       prefixPathOpt,
       tcSink,
       tcState: TcState,
-      inp: ParsedInput,
+      input: ParsedInput,
       _skipImplIfSigExists: bool): (unit -> bool) * TcConfig * TcImports * TcGlobals * LongIdent option * TcResultsSink * TcState * ParsedInput * bool)
     : Cancellable<Finisher<TcState, PartialResult>> =
     cancellable {
         try
             CheckSimulateException tcConfig
 
-            let m = inp.Range
+            let m = input.Range
             let amap = tcImports.GetImportMap()
 
             let conditionalDefines =
@@ -1483,7 +1483,7 @@ let CheckOneInputWithCallback
                 else
                     Some tcConfig.conditionalDefines
 
-            match inp with
+            match input with
             | ParsedInput.SigFile file ->
                 let qualNameOfFile = file.QualifiedName
 
@@ -1609,6 +1609,43 @@ let AddSignatureResultToTcImplEnv (tcImports: TcImports, tcGlobals, prefixPathOp
 
             partialResult, tcState
 
+let TransformDependencyGraph (graph: Graph<FileIndex>, filePairs: FilePairMap) =
+    let mkArtificialImplFile n = NodeToTypeCheck.ArtificialImplFile n
+    let mkPhysicalFile n = NodeToTypeCheck.PhysicalFile n
+
+    /// Map any signature dependencies to the ArtificialImplFile counterparts,
+    /// unless the signature dependency is the backing file of the current (implementation) file.
+    let mapDependencies idx deps =
+        Array.map
+            (fun dep ->
+                if filePairs.IsSignature dep then
+                    let implIdx = filePairs.GetImplementationIndex dep
+
+                    if implIdx = idx then
+                        // This is the matching signature for the implementation.
+                        // Retain the direct dependency onto the signature file.
+                        mkPhysicalFile dep
+                    else
+                        mkArtificialImplFile dep
+                else
+                    mkPhysicalFile dep)
+            deps
+
+    // Transform the graph to include ArtificialImplFile nodes when necessary.
+    graph
+    |> Seq.collect (fun (KeyValue (fileIdx, deps)) ->
+        if filePairs.IsSignature fileIdx then
+            // Add an additional ArtificialImplFile node for the signature file.
+            [|
+                // Mark the current file as physical and map the dependencies.
+                mkPhysicalFile fileIdx, mapDependencies fileIdx deps
+                // Introduce a new node that depends on the signature.
+                mkArtificialImplFile fileIdx, [| mkPhysicalFile fileIdx |]
+            |]
+        else
+            [| mkPhysicalFile fileIdx, mapDependencies fileIdx deps |])
+    |> Graph.make
+
 /// Constructs a file dependency graph and type-checks the files in parallel where possible.
 let CheckMultipleInputsUsingGraphMode
     ((ctok, checkForErrors, tcConfig: TcConfig, tcImports: TcImports, tcGlobals, prefixPathOpt, tcState, eagerFormat, inputs): 'a * (unit -> bool) * TcConfig * TcImports * TcGlobals * LongIdent option * TcState * (PhasedDiagnostic -> PhasedDiagnostic) * ParsedInput list)
@@ -1630,42 +1667,7 @@ let CheckMultipleInputsUsingGraphMode
     let graph, trie =
         DependencyResolution.mkGraph tcConfig.compilingFSharpCore filePairs sourceFiles
 
-    let nodeGraph =
-        let mkArtificialImplFile n = NodeToTypeCheck.ArtificialImplFile n
-        let mkPhysicalFile n = NodeToTypeCheck.PhysicalFile n
-
-        /// Map any signature dependencies to the ArtificialImplFile counterparts,
-        /// unless the signature dependency is the backing file of the current (implementation) file.
-        let mapDependencies idx deps =
-            Array.map
-                (fun dep ->
-                    if filePairs.IsSignature dep then
-                        let implIdx = filePairs.GetImplementationIndex dep
-
-                        if implIdx = idx then
-                            // This is the matching signature for the implementation.
-                            // Retain the direct dependency onto the signature file.
-                            mkPhysicalFile dep
-                        else
-                            mkArtificialImplFile dep
-                    else
-                        mkPhysicalFile dep)
-                deps
-
-        // Transform the graph to include ArtificialImplFile nodes when necessary.
-        graph
-        |> Seq.collect (fun (KeyValue (fileIdx, deps)) ->
-            if filePairs.IsSignature fileIdx then
-                // Add an additional ArtificialImplFile node for the signature file.
-                [|
-                    // Mark the current file as physical and map the dependencies.
-                    mkPhysicalFile fileIdx, mapDependencies fileIdx deps
-                    // Introduce a new node that depends on the signature.
-                    mkArtificialImplFile fileIdx, [| mkPhysicalFile fileIdx |]
-                |]
-            else
-                [| mkPhysicalFile fileIdx, mapDependencies fileIdx deps |])
-        |> Graph.make
+    let nodeGraph = TransformDependencyGraph(graph, filePairs)
 
     // Persist the graph to a Mermaid diagram if specified.
     if tcConfig.typeCheckingConfig.DumpGraph then
@@ -1685,7 +1687,7 @@ let CheckMultipleInputsUsingGraphMode
                         .TrimStart([| '\\'; '/' |])
 
                 (idx, friendlyFileName))
-            |> Graph.serialiseToMermaid graphFile)
+            |> Graph.writeMermaidToFile graphFile)
 
     let _ = ctok // TODO Use it
     let diagnosticsLogger = DiagnosticsThreadStatics.DiagnosticsLogger

src/Compiler/Driver/ParseAndCheckInputs.fsi

@@ -13,12 +13,44 @@ open FSharp.Compiler.CompilerImports
 open FSharp.Compiler.Diagnostics
 open FSharp.Compiler.DependencyManager
 open FSharp.Compiler.DiagnosticsLogger
+open FSharp.Compiler.GraphChecking
+open FSharp.Compiler.NameResolution
 open FSharp.Compiler.Syntax
 open FSharp.Compiler.TcGlobals
 open FSharp.Compiler.Text
 open FSharp.Compiler.TypedTree
 open FSharp.Compiler.UnicodeLexing
 
+/// Auxiliary type for re-using signature information in TcEnvFromImpls.
+///
+/// TcState has two typing environments: TcEnvFromSignatures && TcEnvFromImpls
+/// When type checking a file, depending on the type (implementation or signature), it will use one of these typing environments (TcEnv).
+/// Checking a file will populate the respective TcEnv.
+///
+/// When a file has a dependencies, the information of the signature file in case a pair (implementation file backed by a signature) will suffice to type-check that file.
+/// Example: if `B.fs` has a dependency on `A`, the information of `A.fsi` is enough for `B.fs` to type-check, on condition that information is available in the TcEnvFromImpls.
+/// We introduce a special ArtificialImplFile node in the graph to satisfy this. `B.fs -> [ A.fsi ]` becomes `B.fs -> [ ArtificialImplFile A ].
+/// The `ArtificialImplFile A` node will duplicate the signature information which A.fsi provided earlier.
+/// Processing a `ArtificialImplFile` node will add the information from the TcEnvFromSignatures to the TcEnvFromImpls.
+/// This means `A` will be known in both TcEnvs and therefor `B.fs` can be type-checked.
+/// By doing this, we can speed up the graph processing as type checking a signature file is less expensive than its implementation counterpart.
+///
+/// When we need to actually type-check an implementation file backed by a signature, we cannot have the duplicate information of the signature file present in TcEnvFromImpls.
+/// Example `A.fs -> [ A.fsi ]`. An implementation file always depends on its signature.
+/// Type-checking `A.fs` will add the actual information to TcEnvFromImpls and we do not depend on the `ArtificialImplFile A` for `A.fs`.
+///
+/// In order to deal correctly with the `ArtificialImplFile` logic, we need to transform the resolved graph to contain the additional pair nodes.
+/// After we have type-checked the graph, we exclude the ArtificialImplFile nodes as they are not actual physical files in our project.
+[<RequireQualifiedAccess>]
+type NodeToTypeCheck =
+    /// A real physical file in the current project.
+    /// This can be either an implementation or a signature file.
+    | PhysicalFile of fileIndex: FileIndex
+    /// An artificial node that will add the earlier processed signature information to the TcEnvFromImpls.
+    /// Dependants on this type of node will perceive that a file is known in both TcEnvFromSignatures and TcEnvFromImpls.
+    /// Even though the actual implementation file was not type-checked.
+    | ArtificialImplFile of signatureFileIndex: FileIndex
+
 val IsScript: string -> bool
 
 val ComputeQualifiedNameOfFileFromUniquePath: range * string list -> QualifiedNameOfFile
@@ -131,6 +163,8 @@ type TcState =
 
     member CreatesGeneratedProvidedTypes: bool
 
+type PartialResult = TcEnv * TopAttribs * CheckedImplFile option * ModuleOrNamespaceType
+
 /// Get the initial type checking state for a set of inputs
 val GetInitialTcState: range * string * TcConfig * TcGlobals * TcImports * TcEnv * OpenDeclaration list -> TcState
 
@@ -151,6 +185,41 @@ val CheckOneInput:
     input: ParsedInput ->
         Cancellable<(TcEnv * TopAttribs * CheckedImplFile option * ModuleOrNamespaceType) * TcState>
 
+val CheckOneInputWithCallback:
+    checkForErrors: (unit -> bool) *
+    tcConfig: TcConfig *
+    tcImports: TcImports *
+    tcGlobals: TcGlobals *
+    prefixPathOpt: LongIdent option *
+    tcSink: TcResultsSink *
+    tcState: TcState *
+    input: ParsedInput *
+    _skipImplIfSigExists: bool ->
+        Cancellable<Finisher<TcState, PartialResult>>
+
+val AddCheckResultsToTcState:
+    tcGlobals: TcGlobals *
+    amap: Import.ImportMap *
+    hadSig: bool *
+    prefixPathOpt: LongIdent option *
+    tcSink: TcResultsSink *
+    tcImplEnv: TcEnv *
+    qualNameOfFile: QualifiedNameOfFile *
+    implFileSigType: ModuleOrNamespaceType ->
+        tcState: TcState ->
+            ModuleOrNamespaceType * TcState
+
+val AddSignatureResultToTcImplEnv:
+    tcImports: TcImports *
+    tcGlobals: TcGlobals *
+    prefixPathOpt: LongIdent option *
+    tcSink: TcResultsSink *
+    tcState: TcState *
+    input: ParsedInput ->
+        (TcState -> PartialResult * TcState)
+
+val TransformDependencyGraph: graph: Graph<FileIndex> * filePairs: FilePairMap -> Graph<NodeToTypeCheck>
+
 /// Finish the checking of multiple inputs
 val CheckMultipleInputsFinish:
     (TcEnv * TopAttribs * 'T option * 'U) list * TcState -> (TcEnv * TopAttribs * 'T list * 'U list) * TcState

src/Compiler/FSharp.Compiler.Service.fsproj

@@ -154,6 +154,7 @@
     <Compile Include="Facilities\CompilerLocation.fs" />
     <Compile Include="Facilities\BuildGraph.fsi" />
     <Compile Include="Facilities\BuildGraph.fs" />
+    <Compile Include="Facilities\AsyncMemoize.fs" />
     <FsLex Include="AbstractIL\illex.fsl">
       <OtherFlags>--module FSharp.Compiler.AbstractIL.AsciiLexer --internal --open Internal.Utilities.Text.Lexing --open FSharp.Compiler.AbstractIL.AsciiParser --unicode --lexlib Internal.Utilities.Text.Lexing</OtherFlags>
       <Link>AbstractIL\illex.fsl</Link>
@@ -486,7 +487,9 @@
     <Compile Include="Service\QuickParse.fsi" />
     <Compile Include="Service\QuickParse.fs" />
     <Compile Include="Service\FSharpCheckerResults.fsi" />
-    <Compile Include="Service\FSharpCheckerResults.fs" />
+	  <Compile Include="Service\FSharpCheckerResults.fs" />
+	  <Compile Include="Service\BackgroundCompiler.fs" />
+	  <Compile Include="Service\TransparentCompiler.fs" />
     <Compile Include="Service\service.fsi" />
     <Compile Include="Service\service.fs" />
     <Compile Include="Service\ServiceInterfaceStubGenerator.fsi" />