Refactor BufferLocalStore related code

.fantomasignore

@@ -116,6 +116,7 @@ src/Compiler/Utilities/HashMultiMap.fs
 src/Compiler/Facilities/AsyncMemoize.fsi
 src/Compiler/Facilities/AsyncMemoize.fs
 src/Compiler/AbstractIL/il.fs
+src/Compiler/SyntaxTree/LexerStore.fs
 
 src/Compiler/Driver/GraphChecking/Graph.fsi
 src/Compiler/Driver/GraphChecking/Graph.fs

src/Compiler/Driver/ParseAndCheckInputs.fs

@@ -27,6 +27,7 @@ open FSharp.Compiler.Diagnostics
 open FSharp.Compiler.DiagnosticsLogger
 open FSharp.Compiler.Features
 open FSharp.Compiler.IO
+open FSharp.Compiler.LexerStore
 open FSharp.Compiler.Lexhelp
 open FSharp.Compiler.NameResolution
 open FSharp.Compiler.ParseHelpers
@@ -239,7 +240,7 @@ let GetScopedPragmasForHashDirective hd (langVersion: LanguageVersion) =
 
 let private collectCodeComments (lexbuf: UnicodeLexing.Lexbuf) (tripleSlashComments: range list) =
     [
-        yield! LexbufCommentStore.GetComments(lexbuf)
+        yield! CommentStore.GetComments(lexbuf)
         yield! (List.map CommentTrivia.LineComment tripleSlashComments)
     ]
     |> List.sortBy (function
@@ -285,7 +286,7 @@ let PostParseModuleImpls
                 yield! GetScopedPragmasForHashDirective hd lexbuf.LanguageVersion
         ]
 
-    let conditionalDirectives = LexbufIfdefStore.GetTrivia(lexbuf)
+    let conditionalDirectives = IfdefStore.GetTrivia(lexbuf)
     let codeComments = collectCodeComments lexbuf tripleSlashComments
 
     let trivia: ParsedImplFileInputTrivia =
@@ -336,7 +337,7 @@ let PostParseModuleSpecs
                 yield! GetScopedPragmasForHashDirective hd lexbuf.LanguageVersion
         ]
 
-    let conditionalDirectives = LexbufIfdefStore.GetTrivia(lexbuf)
+    let conditionalDirectives = IfdefStore.GetTrivia(lexbuf)
     let codeComments = collectCodeComments lexbuf tripleSlashComments
 
     let trivia: ParsedSigFileInputTrivia =
@@ -488,15 +489,13 @@ let ParseInput
             if FSharpImplFileSuffixes |> List.exists (FileSystemUtils.checkSuffix fileName) then
                 let impl = Parser.implementationFile lexer lexbuf
 
-                let tripleSlashComments =
-                    LexbufLocalXmlDocStore.ReportInvalidXmlDocPositions(lexbuf)
+                let tripleSlashComments = XmlDocStore.ReportInvalidXmlDocPositions(lexbuf)
 
                 PostParseModuleImpls(defaultNamespace, fileName, isLastCompiland, impl, lexbuf, tripleSlashComments, Set identStore)
             elif FSharpSigFileSuffixes |> List.exists (FileSystemUtils.checkSuffix fileName) then
                 let intfs = Parser.signatureFile lexer lexbuf
 
-                let tripleSlashComments =
-                    LexbufLocalXmlDocStore.ReportInvalidXmlDocPositions(lexbuf)
+                let tripleSlashComments = XmlDocStore.ReportInvalidXmlDocPositions(lexbuf)
 
                 PostParseModuleSpecs(defaultNamespace, fileName, isLastCompiland, intfs, lexbuf, tripleSlashComments, Set identStore)
             else if lexbuf.SupportsFeature LanguageFeature.MLCompatRevisions then

src/Compiler/FSharp.Compiler.Service.fsproj

@@ -247,11 +247,11 @@
       <Link>SyntaxTree\pplex.fsl</Link>
     </FsLex>
     <FsYacc Include="pppars.fsy">
-      <OtherFlags>--module FSharp.Compiler.PPParser --open FSharp.Compiler.ParseHelpers --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing --buffer-type-argument char</OtherFlags>
+      <OtherFlags>--module FSharp.Compiler.PPParser --open FSharp.Compiler.ParseHelpers --open FSharp.Compiler.LexerStore --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing --buffer-type-argument char</OtherFlags>
       <Link>SyntaxTree\pppars.fsy</Link>
     </FsYacc>
     <FsLex Include="lex.fsl">
-      <OtherFlags>--module FSharp.Compiler.Lexer --open FSharp.Compiler.Lexhelp --open Internal.Utilities.Text.Lexing --open FSharp.Compiler.Parser --open FSharp.Compiler.Text --open FSharp.Compiler.ParseHelpers --internal --unicode --lexlib Internal.Utilities.Text.Lexing</OtherFlags>
+      <OtherFlags>--module FSharp.Compiler.Lexer --open FSharp.Compiler.Lexhelp --open Internal.Utilities.Text.Lexing --open FSharp.Compiler.Parser --open FSharp.Compiler.Text --open FSharp.Compiler.ParseHelpers --open FSharp.Compiler.LexerStore --internal --unicode --lexlib Internal.Utilities.Text.Lexing</OtherFlags>
       <Link>SyntaxTree\lex.fsl</Link>
     </FsLex>
     <FsYacc Include="pars.fsy">
@@ -280,6 +280,8 @@
     <Compile Include="SyntaxTree\SyntaxTree.fs" />
     <Compile Include="SyntaxTree\SyntaxTreeOps.fsi" />
     <Compile Include="SyntaxTree\SyntaxTreeOps.fs" />
+    <Compile Include="SyntaxTree\LexerStore.fsi" />
+    <Compile Include="SyntaxTree\LexerStore.fs" />
     <Compile Include="SyntaxTree\ParseHelpers.fsi" />
     <Compile Include="SyntaxTree\ParseHelpers.fs" />
     <Compile Include="$(FsYaccOutputFolder)pppars.fsi">

src/Compiler/SyntaxTree/LexFilter.fs

@@ -7,11 +7,11 @@ module internal FSharp.Compiler.LexFilter
 open System
 open System.Collections.Generic
 open Internal.Utilities.Text.Lexing
-open FSharp.Compiler
 open Internal.Utilities.Library
 open FSharp.Compiler.AbstractIL.Diagnostics
 open FSharp.Compiler.DiagnosticsLogger
 open FSharp.Compiler.Features
+open FSharp.Compiler.LexerStore
 open FSharp.Compiler.Lexhelp
 open FSharp.Compiler.ParseHelpers
 open FSharp.Compiler.Parser
@@ -665,7 +665,7 @@ type LexFilterImpl (
         let lastTokenEnd = state.EndPos
         let token = lexer lexbuf
 
-        LexbufLocalXmlDocStore.AddGrabPoint(lexbuf)
+        XmlDocStore.AddGrabPoint(lexbuf)
 
         // Now we've got the token, remember the lexbuf state, associating it with the token
         // and remembering it as the last observed lexbuf state for the wrapped lexer function.

src/Compiler/SyntaxTree/LexHelpers.fs

@@ -12,6 +12,7 @@ open Internal.Utilities.Text.Lexing
 open FSharp.Compiler.IO
 open FSharp.Compiler.DiagnosticsLogger
 open FSharp.Compiler.Features
+open FSharp.Compiler.LexerStore
 open FSharp.Compiler.ParseHelpers
 open FSharp.Compiler.UnicodeLexing
 open FSharp.Compiler.Parser
@@ -98,10 +99,8 @@ let mkLexargs
     }
 
 /// Register the lexbuf and call the given function
-let reusingLexbufForParsing lexbuf f =
+let reusingLexbufForParsing (lexbuf: Lexbuf) f =
     use _ = UseBuildPhase BuildPhase.Parse
-    LexbufLocalXmlDocStore.ClearXmlDoc lexbuf
-    LexbufCommentStore.ClearComments lexbuf
 
     try
         f ()

src/Compiler/SyntaxTree/LexerStore.fs

@@ -0,0 +1,169 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
+
+module internal FSharp.Compiler.LexerStore
+
+open FSharp.Compiler.SyntaxTreeOps
+open FSharp.Compiler.SyntaxTrivia
+open FSharp.Compiler.UnicodeLexing
+open FSharp.Compiler.Text
+open FSharp.Compiler.Text.Position
+open FSharp.Compiler.Text.Range
+open FSharp.Compiler.Xml
+
+//------------------------------------------------------------------------
+// Lexbuf.BufferLocalStore is used during lexing/parsing of a file for different purposes.
+// All access happens through the functions and modules below.
+//------------------------------------------------------------------------
+
+let private getStoreData<'T when 'T: not null> (lexbuf: Lexbuf) key (getInitialData: unit -> 'T) =
+    let store = lexbuf.BufferLocalStore
+
+    match store.TryGetValue key with
+    | true, data -> data :?> 'T
+    | _ ->
+        let data = getInitialData ()
+        store[key] <- data
+        data
+
+let private tryGetStoreData<'T when 'T: not null> (lexbuf: Lexbuf) key =
+    let store = lexbuf.BufferLocalStore
+
+    match store.TryGetValue key with
+    | true, data -> Some(data :?> 'T)
+    | _ -> None
+
+let private setStoreData (lexbuf: Lexbuf) key data = lexbuf.BufferLocalStore[key] <- data
+
+//------------------------------------------------------------------------
+// A SynArgNameGenerator for the current file, used by the parser
+//------------------------------------------------------------------------
+
+let getSynArgNameGenerator (lexbuf: Lexbuf) =
+    getStoreData lexbuf "SynArgNameGenerator" SynArgNameGenerator
+
+//------------------------------------------------------------------------
+// A XmlDocCollector, used to hold the current accumulated Xml doc lines, and related access functions
+//------------------------------------------------------------------------
+
+[<RequireQualifiedAccess>]
+module XmlDocStore =
+    let private xmlDocKey = "XmlDoc"
+
+    let private getCollector (lexbuf: Lexbuf) =
+        getStoreData lexbuf xmlDocKey XmlDocCollector
+
+    /// Called from the lexer to save a single line of XML doc comment.
+    let SaveXmlDocLine (lexbuf: Lexbuf, lineText, range: range) =
+        let collector = getCollector lexbuf
+        collector.AddXmlDocLine(lineText, range)
+
+    let AddGrabPoint (lexbuf: Lexbuf) =
+        let collector = getCollector lexbuf
+        let startPos = lexbuf.StartPos
+        collector.AddGrabPoint(mkPos startPos.Line startPos.Column)
+
+    /// Allowed cases when there are comments after XmlDoc
+    ///
+    ///    /// X xmlDoc
+    ///    // comment
+    ///    //// comment
+    ///    (* multiline comment *)
+    ///    let x = ...        // X xmlDoc
+    ///
+    /// Remember the first position when a comment (//, (* *), ////) is encountered after the XmlDoc block
+    /// then add a grab point if a new XmlDoc block follows the comments
+    let AddGrabPointDelayed (lexbuf: Lexbuf) =
+        let collector = getCollector lexbuf
+        let startPos = lexbuf.StartPos
+        collector.AddGrabPointDelayed(mkPos startPos.Line startPos.Column)
+
+    /// Called from the parser each time we parse a construct that marks the end of an XML doc comment range,
+    /// e.g. a 'type' declaration. The markerRange is the range of the keyword that delimits the construct.
+    let GrabXmlDocBeforeMarker (lexbuf: Lexbuf, markerRange: range) =
+        match tryGetStoreData lexbuf xmlDocKey with
+        | Some collector -> PreXmlDoc.CreateFromGrabPoint(collector, markerRange.Start)
+        | _ -> PreXmlDoc.Empty
+
+    let ReportInvalidXmlDocPositions (lexbuf: Lexbuf) =
+        let collector = getCollector lexbuf
+        collector.CheckInvalidXmlDocPositions()
+
+//------------------------------------------------------------------------
+// Storage to hold the current accumulated ConditionalDirectiveTrivia, and related types and access functions
+//------------------------------------------------------------------------
+
+type LexerIfdefExpression =
+    | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression
+    | IfdefOr of LexerIfdefExpression * LexerIfdefExpression
+    | IfdefNot of LexerIfdefExpression
+    | IfdefId of string
+
+let rec LexerIfdefEval (lookup: string -> bool) =
+    function
+    | IfdefAnd(l, r) -> (LexerIfdefEval lookup l) && (LexerIfdefEval lookup r)
+    | IfdefOr(l, r) -> (LexerIfdefEval lookup l) || (LexerIfdefEval lookup r)
+    | IfdefNot e -> not (LexerIfdefEval lookup e)
+    | IfdefId id -> lookup id
+
+[<RequireQualifiedAccess>]
+module IfdefStore =
+    let private getStore (lexbuf: Lexbuf) =
+        getStoreData lexbuf "Ifdef" ResizeArray<ConditionalDirectiveTrivia>
+
+    let private mkRangeWithoutLeadingWhitespace (lexed: string) (m: range) : range =
+        let startColumn = lexed.Length - lexed.TrimStart().Length
+        mkFileIndexRange m.FileIndex (mkPos m.StartLine startColumn) m.End
+
+    let SaveIfHash (lexbuf: Lexbuf, lexed: string, expr: LexerIfdefExpression, range: range) =
+        let store = getStore lexbuf
+
+        let expr =
+            let rec visit (expr: LexerIfdefExpression) : IfDirectiveExpression =
+                match expr with
+                | LexerIfdefExpression.IfdefAnd(l, r) -> IfDirectiveExpression.And(visit l, visit r)
+                | LexerIfdefExpression.IfdefOr(l, r) -> IfDirectiveExpression.Or(visit l, visit r)
+                | LexerIfdefExpression.IfdefNot e -> IfDirectiveExpression.Not(visit e)
+                | LexerIfdefExpression.IfdefId id -> IfDirectiveExpression.Ident id
+
+            visit expr
+
+        let m = mkRangeWithoutLeadingWhitespace lexed range
+
+        store.Add(ConditionalDirectiveTrivia.If(expr, m))
+
+    let SaveElseHash (lexbuf: Lexbuf, lexed: string, range: range) =
+        let store = getStore lexbuf
+        let m = mkRangeWithoutLeadingWhitespace lexed range
+        store.Add(ConditionalDirectiveTrivia.Else(m))
+
+    let SaveEndIfHash (lexbuf: Lexbuf, lexed: string, range: range) =
+        let store = getStore lexbuf
+        let m = mkRangeWithoutLeadingWhitespace lexed range
+        store.Add(ConditionalDirectiveTrivia.EndIf(m))
+
+    let GetTrivia (lexbuf: Lexbuf) : ConditionalDirectiveTrivia list =
+        let store = getStore lexbuf
+        Seq.toList store
+
+//------------------------------------------------------------------------
+// Storage to hold the current accumulated CommentTrivia, and related access functions
+//------------------------------------------------------------------------
+
+[<RequireQualifiedAccess>]
+module CommentStore =
+    let private getStore (lexbuf: Lexbuf) =
+        getStoreData lexbuf "Comments" ResizeArray<CommentTrivia>
+
+    let SaveSingleLineComment (lexbuf: Lexbuf, startRange: range, endRange: range) =
+        let store = getStore lexbuf
+        let m = unionRanges startRange endRange
+        store.Add(CommentTrivia.LineComment(m))
+
+    let SaveBlockComment (lexbuf: Lexbuf, startRange: range, endRange: range) =
+        let store = getStore lexbuf
+        let m = unionRanges startRange endRange
+        store.Add(CommentTrivia.BlockComment(m))
+
+    let GetComments (lexbuf: Lexbuf) : CommentTrivia list =
+        let store = getStore lexbuf
+        Seq.toList store

src/Compiler/SyntaxTree/LexerStore.fsi

@@ -0,0 +1,52 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
+
+module internal FSharp.Compiler.LexerStore
+
+open FSharp.Compiler.SyntaxTreeOps
+open FSharp.Compiler.SyntaxTrivia
+open FSharp.Compiler.UnicodeLexing
+open FSharp.Compiler.Text
+open FSharp.Compiler.Xml
+
+val getSynArgNameGenerator: Lexbuf -> SynArgNameGenerator
+
+[<RequireQualifiedAccess>]
+module XmlDocStore =
+
+    val SaveXmlDocLine: lexbuf: Lexbuf * lineText: string * range: range -> unit
+
+    val GrabXmlDocBeforeMarker: lexbuf: Lexbuf * markerRange: range -> PreXmlDoc
+
+    val AddGrabPoint: lexbuf: Lexbuf -> unit
+
+    val AddGrabPointDelayed: lexbuf: Lexbuf -> unit
+
+    val ReportInvalidXmlDocPositions: lexbuf: Lexbuf -> range list
+
+type LexerIfdefExpression =
+    | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression
+    | IfdefOr of LexerIfdefExpression * LexerIfdefExpression
+    | IfdefNot of LexerIfdefExpression
+    | IfdefId of string
+
+val LexerIfdefEval: lookup: (string -> bool) -> _arg1: LexerIfdefExpression -> bool
+
+[<RequireQualifiedAccess>]
+module IfdefStore =
+
+    val SaveIfHash: lexbuf: Lexbuf * lexed: string * expr: LexerIfdefExpression * range: range -> unit
+
+    val SaveElseHash: lexbuf: Lexbuf * lexed: string * range: range -> unit
+
+    val SaveEndIfHash: lexbuf: Lexbuf * lexed: string * range: range -> unit
+
+    val GetTrivia: lexbuf: Lexbuf -> ConditionalDirectiveTrivia list
+
+[<RequireQualifiedAccess>]
+module CommentStore =
+
+    val SaveSingleLineComment: lexbuf: Lexbuf * startRange: range * endRange: range -> unit
+
+    val SaveBlockComment: lexbuf: Lexbuf * startRange: range * endRange: range -> unit
+
+    val GetComments: lexbuf: Lexbuf -> CommentTrivia list