Skip to content

Commit

Permalink
LOH allocation fixes for strings and char arrays (#7972)
Browse files Browse the repository at this point in the history 
* Removed UnicodeFileAsLexbuf. Replaced it with StreamReaderAsLexbuf. No longer allocating string and chars as a result.

* Make these private
  • Loading branch information
@TIHan @cartermp
TIHan authored and cartermp committed Dec 16, 2019
1 parent dad70de commit 24407d5
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 54 deletions.
43 changes: 42 additions & 1 deletion src/absil/illib.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1361,7 +1361,45 @@ module Shim =
directory.Contains("packages\\") ||
directory.Contains("lib/mono/")

let mutable FileSystem = DefaultFileSystem() :> IFileSystem
let mutable FileSystem = DefaultFileSystem() :> IFileSystem

// The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure
// uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold
// plus 1 second. Once past the threshold the incremental builder will be able to retry asynchronously based
// on plain old timestamp checking.
//
// The sleep time of 50ms is chosen so that we can respond to the user more quickly for Intellisense operations.
//
// This is not run on the UI thread for VS but it is on a thread that must be stopped before Intellisense
// can return any result except for pending.
let private retryDelayMilliseconds = 50
let private numRetries = 60

let private getReader (filename, codePage: int option, retryLocked: bool) =
// Retry multiple times since other processes may be writing to this file.
let rec getSource retryNumber =
try
// Use the .NET functionality to auto-detect the unicode encoding
let stream = FileSystem.FileStreamReadShim(filename)
match codePage with
| None -> new StreamReader(stream,true)
| Some n -> new StreamReader(stream,System.Text.Encoding.GetEncoding(n))
with
// We can get here if the file is locked--like when VS is saving a file--we don't have direct
// access to the HRESULT to see that this is EONOACCESS.
| :? System.IO.IOException as err when retryLocked && err.GetType() = typeof<System.IO.IOException> ->
// This second check is to make sure the exception is exactly IOException and none of these for example:
// DirectoryNotFoundException
// EndOfStreamException
// FileNotFoundException
// FileLoadException
// PathTooLongException
if retryNumber < numRetries then
System.Threading.Thread.Sleep (retryDelayMilliseconds)
getSource (retryNumber + 1)
else
reraise()
getSource 0

type File with

Expand All @@ -1374,3 +1412,6 @@ module Shim =
n <- n + stream.Read(buffer, n, len-n)
buffer

static member OpenReaderAndRetry (filename, codepage, retryLocked) =
getReader (filename, codepage, retryLocked)

5 changes: 3 additions & 2 deletions src/fsharp/CompileOps.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3436,7 +3436,7 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp
if verbose then dprintn ("Parsed "+shortFilename)
Some input
with e -> (* errorR(Failure("parse failed")); *) errorRecovery e rangeStartup; None


let ParseOneInputFile (tcConfig: TcConfig, lexResourceManager, conditionalCompilationDefines, filename, isLastCompiland, errorLogger, retryLocked) =
try
Expand All @@ -3445,7 +3445,8 @@ let ParseOneInputFile (tcConfig: TcConfig, lexResourceManager, conditionalCompil
if not(FileSystem.SafeExists filename) then
error(Error(FSComp.SR.buildCouldNotFindSourceFile filename, rangeStartup))
let isFeatureSupported featureId = tcConfig.langVersion.SupportsFeature featureId
let lexbuf = UnicodeLexing.UnicodeFileAsLexbuf(isFeatureSupported, filename, tcConfig.inputCodePage, retryLocked)
use reader = File.OpenReaderAndRetry (filename, tcConfig.inputCodePage, retryLocked)
let lexbuf = UnicodeLexing.StreamReaderAsLexbuf(isFeatureSupported, reader)
ParseOneInputLexbuf(tcConfig, lexResourceManager, conditionalCompilationDefines, lexbuf, filename, isLastCompiland, errorLogger)
else error(Error(FSComp.SR.buildInvalidSourceFileExtension(SanitizeFileName filename tcConfig.implicitIncludeDir), rangeStartup))
with e -> (* errorR(Failure("parse failed")); *) errorRecovery e rangeStartup; None
Expand Down
59 changes: 12 additions & 47 deletions src/fsharp/UnicodeLexing.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,50 +22,15 @@ let FunctionAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, bufferF
let SourceTextAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, sourceText) =
LexBuffer<char>.FromSourceText(supportsFeature, sourceText)

// The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure
// uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold
// plus 1 second. Once past the threshold the incremental builder will be able to retry asynchronously based
// on plain old timestamp checking.
//
// The sleep time of 50ms is chosen so that we can respond to the user more quickly for Intellisense operations.
//
// This is not run on the UI thread for VS but it is on a thread that must be stopped before Intellisense
// can return any result except for pending.
let retryDelayMilliseconds = 50
let numRetries = 60

/// Standard utility to create a Unicode LexBuffer
///
/// One small annoyance is that LexBuffers and not IDisposable. This means
/// we can't just return the LexBuffer object, since the file it wraps wouldn't
/// get closed when we're finished with the LexBuffer. Hence we return the stream,
/// the reader and the LexBuffer. The caller should dispose the first two when done.
let UnicodeFileAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, filename, codePage: int option, retryLocked: bool): Lexbuf =
// Retry multiple times since other processes may be writing to this file.
let rec getSource retryNumber =
try
// Use the .NET functionality to auto-detect the unicode encoding
use stream = FileSystem.FileStreamReadShim(filename)
use reader =
match codePage with
| None -> new StreamReader(stream,true)
| Some n -> new StreamReader(stream,System.Text.Encoding.GetEncoding(n))
reader.ReadToEnd()
with
// We can get here if the file is locked--like when VS is saving a file--we don't have direct
// access to the HRESULT to see that this is EONOACCESS.
| :? System.IO.IOException as err when retryLocked && err.GetType() = typeof<System.IO.IOException> ->
// This second check is to make sure the exception is exactly IOException and none of these for example:
// DirectoryNotFoundException
// EndOfStreamException
// FileNotFoundException
// FileLoadException
// PathTooLongException
if retryNumber < numRetries then
System.Threading.Thread.Sleep (retryDelayMilliseconds)
getSource (retryNumber + 1)
else
reraise()
let source = getSource 0
let lexbuf = LexBuffer<_>.FromChars(supportsFeature, source.ToCharArray())
lexbuf
let StreamReaderAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, reader: StreamReader) =
let mutable isFinished = false
FunctionAsLexbuf (supportsFeature, fun (chars, start, length) ->
if isFinished then 0
else
let nBytesRead = reader.Read(chars, start, length)
if nBytesRead = 0 then
isFinished <- true
0
else
nBytesRead
)
5 changes: 4 additions & 1 deletion src/fsharp/UnicodeLexing.fsi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

module internal FSharp.Compiler.UnicodeLexing

open System.IO
open FSharp.Compiler.Features
open FSharp.Compiler.Text
open Microsoft.FSharp.Text
Expand All @@ -10,5 +11,7 @@ open Internal.Utilities.Text.Lexing
type Lexbuf = LexBuffer<char>
val internal StringAsLexbuf: (Features.LanguageFeature -> bool) * string -> Lexbuf
val public FunctionAsLexbuf: (Features.LanguageFeature -> bool) * (char [] * int * int -> int) -> Lexbuf
val public UnicodeFileAsLexbuf: (Features.LanguageFeature -> bool) * string * int option * (*retryLocked*) bool -> Lexbuf
val public SourceTextAsLexbuf: (Features.LanguageFeature -> bool) * ISourceText -> Lexbuf

/// Will not dispose of the stream reader.
val public StreamReaderAsLexbuf: (Features.LanguageFeature -> bool) * StreamReader -> Lexbuf
7 changes: 4 additions & 3 deletions src/fsharp/fsi/fsi.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1677,8 +1677,8 @@ type internal FsiStdinLexerProvider
CreateLexerForLexBuffer (Lexhelp.stdinMockFilename, lexbuf, errorLogger)

// Create a new lexer to read an "included" script file
member __.CreateIncludedScriptLexer (sourceFileName, errorLogger) =
let lexbuf = UnicodeLexing.UnicodeFileAsLexbuf(isFeatureSupported, sourceFileName, tcConfigB.inputCodePage, (*retryLocked*)false)
member __.CreateIncludedScriptLexer (sourceFileName, reader, errorLogger) =
let lexbuf = UnicodeLexing.StreamReaderAsLexbuf(isFeatureSupported, reader)
CreateLexerForLexBuffer (sourceFileName, lexbuf, errorLogger)

// Create a new lexer to read a string
Expand Down Expand Up @@ -2037,7 +2037,8 @@ type internal FsiInteractionProcessor
WithImplicitHome (tcConfigB, directoryName sourceFile) (fun () ->
// An included script file may contain maybe several interaction blocks.
// We repeatedly parse and process these, until an error occurs.
let tokenizer = fsiStdinLexerProvider.CreateIncludedScriptLexer (sourceFile, errorLogger)
use reader = File.OpenReaderAndRetry (sourceFile, tcConfigB.inputCodePage, (*retryLocked*)false)
let tokenizer = fsiStdinLexerProvider.CreateIncludedScriptLexer (sourceFile, reader, errorLogger)
let rec run istate =
let istate,cont = processor.ParseAndExecOneSetOfInteractionsFromLexbuf ((fun f istate -> f ctok istate), istate, tokenizer, errorLogger)
match cont with Completed _ -> run istate | _ -> istate,cont
Expand Down

0 comments on commit 24407d5

Please sign in to comment.