Skip to content
This repository has been archived by the owner on Jul 19, 2019. It is now read-only.

Commit

Permalink
Added parse and type check caches
Browse files Browse the repository at this point in the history
  • Loading branch information
ncave committed Dec 11, 2018
1 parent cbd2578 commit dc3c88b
Show file tree
Hide file tree
Showing 13 changed files with 335 additions and 126 deletions.
51 changes: 42 additions & 9 deletions fcs/fcs-fable/service_shim.fs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ open Microsoft.FSharp.Compiler.TypeChecker
// InteractiveChecker
//-------------------------------------------------------------------------

type InteractiveChecker internal (tcConfig, tcGlobals, tcImports, tcInitialState, ctok, reactorOps, moduleNamesDict) =
type internal TcResult = TcEnv * TopAttribs * TypedImplFile option * ModuleOrNamespaceType

type InteractiveChecker internal (tcConfig, tcGlobals, tcImports, tcInitialState, ctok, reactorOps, moduleNamesDict, parseCache, checkCache) =
let userOpName = "Unknown"

static member Create(references: string[], readAllBytes: string -> byte[], defines: string[]) =
Expand Down Expand Up @@ -273,10 +275,13 @@ type InteractiveChecker internal (tcConfig, tcGlobals, tcImports, tcInitialState
async.Return (Cancellable.runWithoutCancellation (op ctok))
member __.EnqueueOp (userOpName, opName, opArg, op) = (op ctok) }

// for de-duplicating module names
// dictionary for de-duplicating module names
let moduleNamesDict = ConcurrentDictionary<string, Set<string>>()
// parse and type check caches
let parseCache = ConcurrentDictionary<string * int * FSharpParsingOptions, FSharpParseFileResults>(HashIdentity.Structural)
let checkCache = ConcurrentDictionary<string, TcResult * TcState>(HashIdentity.Structural)

InteractiveChecker (tcConfig, tcGlobals, tcImports, tcInitialState, ctok, reactorOps, moduleNamesDict)
InteractiveChecker (tcConfig, tcGlobals, tcImports, tcInitialState, ctok, reactorOps, moduleNamesDict, parseCache, checkCache)

member private x.MakeProjectResults (projectFileName: string, parseResults: FSharpParseFileResults[], tcState: TcState, errors: FSharpErrorInfo[],
symbolUses: TcSymbolUses list, topAttrsOpt: TopAttribs option, tcImplFilesOpt: TypedImplFile list option) =
Expand All @@ -287,11 +292,39 @@ type InteractiveChecker internal (tcConfig, tcGlobals, tcImports, tcInitialState
let details = (tcGlobals, tcImports, tcState.Ccu, tcState.CcuSig, symbolUses, topAttrsOpt, assemblyDataOpt, assemblyRef, access, tcImplFilesOpt, dependencyFiles)
FSharpCheckProjectResults (projectFileName, Some tcConfig, true, errors, Some details)

member private x.ParseScript (filename: string, source: string, parsingOptions: FSharpParsingOptions) =
let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (source, filename, parsingOptions, userOpName)
let parseTreeOpt = parseTreeOpt |> Option.map (DeduplicateParsedInputModuleName moduleNamesDict)
let dependencyFiles = [||] // interactions have no dependencies
FSharpParseFileResults (parseErrors, parseTreeOpt, anyErrors, dependencyFiles)
member private x.ClearStaleCache (fileName: string, parsingOptions: FSharpParsingOptions) =
let fileIndex = parsingOptions.SourceFiles |> Array.findIndex ((=) fileName)
let _, staleCheckKeys = parsingOptions.SourceFiles |> Array.splitAt fileIndex
let staleParseKeys = parseCache.Keys |> Seq.filter (fun (fname,_,_) -> fname = fileName) |> Seq.toArray
staleParseKeys |> Array.iter (fun key -> parseCache.Remove(key) |> ignore)
staleCheckKeys |> Array.iter (fun key -> checkCache.Remove(key) |> ignore)

member private x.ParseScript (fileName: string, source: string, parsingOptions: FSharpParsingOptions) =
let parseCacheKey = fileName, hash source, parsingOptions
parseCache.GetOrAdd(parseCacheKey, fun _ ->
x.ClearStaleCache(fileName, parsingOptions)
let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (source, fileName, parsingOptions, userOpName)
let parseTreeOpt = parseTreeOpt |> Option.map (DeduplicateParsedInputModuleName moduleNamesDict)
let dependencyFiles = [||] // interactions have no dependencies
FSharpParseFileResults (parseErrors, parseTreeOpt, anyErrors, dependencyFiles) )

member private x.TypeCheckClosedInputSet (ctok, checkForErrors, tcConfig, tcImports, tcGlobals, prefixPathOpt, tcState, inputs) =
// tcEnvAtEndOfLastFile is the environment when incrementally adding definitions
let fileNameOf = function
| ParsedInput.SigFile (ParsedSigFileInput(fileName,_,_,_,_)) -> fileName
| ParsedInput.ImplFile (ParsedImplFileInput(fileName,_,_,_,_,_,_)) -> fileName
let cachedTypeCheck tcState (input: ParsedInput) =
let checkCacheKey = fileNameOf input
checkCache.GetOrAdd(checkCacheKey, fun _ ->
TypeCheckOneInput (ctok, checkForErrors, tcConfig, tcImports, tcGlobals, prefixPathOpt) tcState input)
let results, tcState = (tcState, inputs) ||> List.mapFold cachedTypeCheck
let (tcEnvAtEndOfLastFile, topAttrs, implFiles, _), tcState = TypeCheckMultipleInputsFinish(results, tcState)
let tcState, declaredImpls = TypeCheckClosedInputSetFinish (implFiles, tcState)
tcState, topAttrs, declaredImpls, tcEnvAtEndOfLastFile

member x.ClearCache () =
parseCache.Clear()
checkCache.Clear()

member x.ParseAndCheckScript (projectFileName, filename: string, source: string) =
let parsingOptions = FSharpParsingOptions.FromTcConfig(tcConfig, [| filename |], true)
Expand Down Expand Up @@ -380,7 +413,7 @@ type InteractiveChecker internal (tcConfig, tcGlobals, tcImports, tcInitialState
let checkForErrors() = parseHadErrors
let prefixPathOpt = None
let tcState, topAttrs, tcImplFiles, _tcEnvAtEnd =
TypeCheckClosedInputSet (ctok, checkForErrors, tcConfig, tcImports, tcGlobals, prefixPathOpt, tcInitialState, inputs)
x.TypeCheckClosedInputSet (ctok, checkForErrors, tcConfig, tcImports, tcGlobals, prefixPathOpt, tcInitialState, inputs)

// make project results
let parseErrors = parseResults |> Array.collect (fun p -> p.Errors)
Expand Down
5 changes: 3 additions & 2 deletions fcs/fcs-fable/test/.gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Output
out/
out*/

# Node
node_modules/
package-lock.json
package-lock.json
yarn.lock
62 changes: 62 additions & 0 deletions fcs/fcs-fable/test/Platform.fs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
module Platform

#if DOTNET_FILE_SYSTEM

open System.IO

let readAllBytes metadataPath (fileName:string) = File.ReadAllBytes (metadataPath + fileName)
let readAllText (filePath:string) = File.ReadAllText (filePath, System.Text.Encoding.UTF8)
let writeAllText (filePath:string) (text:string) = File.WriteAllText (filePath, text)

let measureTime (f: 'a -> 'b) x =
let sw = System.Diagnostics.Stopwatch.StartNew()
let res = f x
sw.Stop()
sw.ElapsedMilliseconds, res

#else // !DOTNET_FILE_SYSTEM

open Fable.Core.JsInterop

type private IFileSystem =
abstract readFileSync: string -> byte[]
abstract readFileSync: string * string -> string
abstract writeFileSync: string * string -> unit

type private IProcess =
abstract hrtime: unit -> float []
abstract hrtime: float[] -> float[]

let private File: IFileSystem = importAll "fs"
let private Process: IProcess = importAll "process"

let readAllBytes metadataPath (fileName:string) = File.readFileSync (metadataPath + fileName)
let readAllText (filePath:string) = (File.readFileSync (filePath, "utf8")).TrimStart('\uFEFF')
let writeAllText (filePath:string) (text:string) = File.writeFileSync (filePath, text)

let measureTime (f: 'a -> 'b) x =
let startTime = Process.hrtime()
let res = f x
let elapsed = Process.hrtime(startTime)
int64 (elapsed.[0] * 1e3 + elapsed.[1] / 1e6), res

#endif

module Path =

let Combine (path1: string, path2: string) =
let path1 =
if path1.Length = 0 then path1
else (path1.TrimEnd [|'\\';'/'|]) + "/"
path1 + (path2.TrimStart [|'\\';'/'|])

let GetFileName (path: string) =
let normPath = path.Replace("\\", "/").TrimEnd('/')
let i = normPath.LastIndexOf("/")
path.Substring(i + 1)

let GetDirectoryName (path: string) =
let normPath = path.Replace("\\", "/")
let i = normPath.LastIndexOf("/")
if i < 0 then ""
else path.Substring(0, i)
151 changes: 151 additions & 0 deletions fcs/fcs-fable/test/bench.fs
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
module App

open System.Text.RegularExpressions
open Microsoft.FSharp.Compiler.SourceCodeServices
open Platform

let references = Metadata.references false
let metadataPath = "/temp/repl/metadata2/" // .NET BCL binaries

let parseProjectFile projectPath =
let projectFileName = Path.GetFileName projectPath
let projectText = readAllText projectPath

// remove all comments
let projectText = Regex.Replace(projectText, @"<!--[\s\S]*?-->", "")

// get conditional defines
let definesRegex = @"<DefineConstants[^>]*>([^<]*)<\/DefineConstants[^>]*>"
let defines =
Regex.Matches(projectText, definesRegex)
|> Seq.collect (fun m -> m.Groups.[1].Value.Split(';'))
|> Seq.append ["FABLE_COMPILER"]
|> Seq.map (fun s -> s.Trim())
|> Seq.distinct
|> Seq.except ["$(DefineConstants)"; ""]
|> Seq.toArray

// get project references
let projectRefsRegex = @"<ProjectReference\s+[^>]*Include\s*=\s*(""[^""]*|'[^']*)"
let projectRefs =
Regex.Matches(projectText, projectRefsRegex)
|> Seq.map (fun m -> m.Groups.[1].Value.TrimStart('"').TrimStart(''').Trim().Replace("\\", "/"))
|> Seq.toArray

// replace some variables
let projectText = projectText.Replace(@"$(MSBuildProjectDirectory)", ".")
let projectText = projectText.Replace(@"$(FSharpSourcesRoot)", "../../src")

// get source files
let sourceFilesRegex = @"<Compile\s+[^>]*Include\s*=\s*(""[^""]*|'[^']*)"
let sourceFiles =
Regex.Matches(projectText, sourceFilesRegex)
|> Seq.map (fun m -> m.Groups.[1].Value.TrimStart('"').TrimStart(''').Trim().Replace("\\", "/"))
|> Seq.toArray

(projectFileName, projectRefs, sourceFiles, defines)

let rec parseProject projectPath =
let (projectFileName, projectRefs, sourceFiles, defines) = parseProjectFile projectPath

let projectFileDir = Path.GetDirectoryName projectPath
let isAbsolutePath (path: string) = path.StartsWith("/")
let trimPath (path: string) = path.TrimStart([|'.';'/'|])
let makePath path = if isAbsolutePath path then path else Path.Combine(projectFileDir, path)
let makeName path = Path.Combine(trimPath projectFileDir, trimPath path)

let fileNames = sourceFiles |> Array.map (fun path -> path |> makeName)
let sources = sourceFiles |> Array.map (fun path -> path |> makePath |> readAllText)

let parsedProjects = projectRefs |> Array.map makePath |> Array.map parseProject
let fileNames = fileNames |> Array.append (parsedProjects |> Array.collect (fun (_,x,_,_) -> x))
let sources = sources |> Array.append (parsedProjects |> Array.collect (fun (_,_,x,_) -> x))
let defines = defines |> Array.append (parsedProjects |> Array.collect (fun (_,_,_,x) -> x))

(projectFileName, fileNames, sources, defines |> Array.distinct)

let dedupFileNames fileNames =
let nameSet = System.Collections.Generic.HashSet<string>()
let padName (name: string) =
let pos = name.LastIndexOf(".")
let nm = if pos < 0 then name else name.Substring(0, pos)
let ext = if pos < 0 then "" else name.Substring(pos)
nm + "_" + ext
let rec dedup name =
if nameSet.Contains(name) then
dedup (padName name)
else
nameSet.Add(name) |> ignore
name
fileNames |> Array.map dedup

let printErrors showWarnings (errors: FSharpErrorInfo[]) =
let isWarning (e: FSharpErrorInfo) =
e.Severity = FSharpErrorSeverity.Warning
let printError (e: FSharpErrorInfo) =
let errorType = (if isWarning e then "Warning" else "Error")
printfn "%s (%d,%d--%d,%d): %s: %s" e.FileName e.EndLineAlternate
e.StartColumn e.EndLineAlternate e.EndColumn errorType e.Message
let warnings, errors = errors |> Array.partition isWarning
let hasErrors = not (Array.isEmpty errors)
if showWarnings then
warnings |> Array.iter printError
if hasErrors then
errors |> Array.iter printError
failwith "Too many errors."

let parseFiles projectPath outDir optimized =
// parse project
let (projectFileName, fileNames, sources, defines) = parseProject projectPath

// dedup file names
let fileNames = dedupFileNames fileNames

// create checker
let createChecker () = InteractiveChecker.Create(references, readAllBytes metadataPath, defines)
let ms0, checker = measureTime createChecker ()
printfn "--------------------------------------------"
printfn "InteractiveChecker created in %d ms" ms0

// parse F# files to AST
let parseFSharp () = checker.ParseAndCheckProject_simple(projectFileName, fileNames, sources)
let ms1, parseRes = measureTime parseFSharp ()
printfn "Project: %s, FCS time: %d ms" projectFileName ms1
printfn "--------------------------------------------"
let showWarnings = false // supress warnings for clarity
parseRes.Errors |> printErrors showWarnings

// exclude signature files
let fileNames = fileNames |> Array.filter (fun x -> not (x.EndsWith(".fsi")))

// this is memory intensive, only do it once
let implFiles = if optimized
then parseRes.GetOptimizedAssemblyContents().ImplementationFiles
else parseRes.AssemblyContents.ImplementationFiles

// for each file
for implFile in implFiles do
let count = List.length implFile.Declarations
printfn "%s: %d declarations" (implFile.FileName) count

// printfn "--------------------------------------------"
// let fsAst = implFile.Declarations |> AstPrint.printFSharpDecls "" |> String.concat "\n"
// printfn "%s" fsAst

let parseArguments (argv: string[]) =
let usage = "Usage: bench <PROJECT_PATH> [--options]"
let opts, args = argv |> Array.partition (fun s -> s.StartsWith("--"))
match args with
| [| projectPath |] ->
let outDir = "./out-test"
let optimized = opts |> Array.contains "--optimize-fcs"
parseFiles projectPath outDir optimized
| _ -> printfn "%s" usage

[<EntryPoint>]
let main argv =
try
parseArguments argv
with ex ->
printfn "Error: %A" ex.Message
0
23 changes: 23 additions & 0 deletions fcs/fcs-fable/test/fcs-fable-bench.fsproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
<DefineConstants>$(DefineConstants);DOTNET_FILE_SYSTEM</DefineConstants>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="../fcs-fable.fsproj" />
</ItemGroup>

<ItemGroup>
<Compile Include="Metadata.fs"/>
<Compile Include="Platform.fs"/>
<Compile Include="bench.fs"/>
</ItemGroup>

<ItemGroup>
<PackageReference Include="Fable.Core" Version="2.0.*" />
</ItemGroup>

</Project>
31 changes: 31 additions & 0 deletions fcs/fcs-fable/test/fcs-fable-bench.sln
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.28307.106
MinimumVisualStudioVersion = 10.0.40219.1
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fcs-fable-bench", "fcs-fable-bench.fsproj", "{83F34C34-6804-4436-923E-E2C539AA59F0}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fcs-fable", "..\fcs-fable.fsproj", "{7D5BC9E3-CEE5-4E1B-BF4E-67553841FD37}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{83F34C34-6804-4436-923E-E2C539AA59F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{83F34C34-6804-4436-923E-E2C539AA59F0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{83F34C34-6804-4436-923E-E2C539AA59F0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{83F34C34-6804-4436-923E-E2C539AA59F0}.Release|Any CPU.Build.0 = Release|Any CPU
{7D5BC9E3-CEE5-4E1B-BF4E-67553841FD37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7D5BC9E3-CEE5-4E1B-BF4E-67553841FD37}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7D5BC9E3-CEE5-4E1B-BF4E-67553841FD37}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7D5BC9E3-CEE5-4E1B-BF4E-67553841FD37}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {BC5C2845-7FCA-4814-93C2-F5910096D973}
EndGlobalSection
EndGlobal
3 changes: 2 additions & 1 deletion fcs/fcs-fable/test/fcs-fable-test.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

<ItemGroup>
<Compile Include="Metadata.fs"/>
<Compile Include="app.fs"/>
<Compile Include="Platform.fs"/>
<Compile Include="test.fs"/>
</ItemGroup>

<ItemGroup>
Expand Down
18 changes: 8 additions & 10 deletions fcs/fcs-fable/test/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,15 @@
"author": "",
"license": "",
"scripts": {
"preinstall": "dotnet restore",
"build": "dotnet fable npm-run splitter",
"rollup": "rollup -c",
"splitter": "node ./node_modules/fable-splitter/dist/cli -c splitter.config.js",
"start": "node out/app"
"build-test": "dotnet run -c Release -p ../../../../Fable/src/dotnet/Fable.Compiler npm-splitter",
"splitter": "node ./node_modules/fable-splitter/dist/cli --commonjs",
"test-node": "node out-test/test",
"test-dotnet": "dotnet run -c Release -p fcs-fable-test.fsproj",
"bench-dotnet": "dotnet run -c Release -p fcs-fable-bench.fsproj ../fcs-fable.fsproj"
},
"devDependencies": {
"@babel/core": "^7.1.2",
"babel-preset-env": "^1.7.0",
"fable-splitter": "^2.0.2",
"rollup": "^0.66.6",
"rollup-plugin-fable": "^2.0.0"
"@babel/core": "^7.2.0",
"@babel/plugin-transform-modules-commonjs": "^7.2.0",
"fable-splitter": "2.0.2"
}
}
Loading

0 comments on commit dc3c88b

Please sign in to comment.