From b0fac9d44bd3d17801ee2d30e86363aa97247a0b Mon Sep 17 00:00:00 2001 From: Phillip Carter Date: Thu, 5 Nov 2020 19:23:08 -0800 Subject: [PATCH] Sequentialize GetAllUsesOfAllSymolsInFile (#10357) --- src/fsharp/service/FSharpCheckerResults.fs | 18 +- src/fsharp/service/FSharpCheckerResults.fsi | 2 +- src/fsharp/service/ServiceAnalysis.fs | 199 +++---- src/fsharp/service/ServiceAnalysis.fsi | 6 +- .../SurfaceArea.netstandard.fs | 6 +- tests/benchmarks/Benchmarks.sln | 28 +- .../CompilerServiceBenchmarks.fsproj | 7 +- .../CompilerServiceBenchmarks/Program.fs | 65 ++- .../decentlySizedStandAloneFile.fsx | 547 ++++++++++++++++++ tests/service/Common.fs | 16 +- tests/service/EditorTests.fs | 8 + tests/service/ProjectAnalysisTests.fs | 6 +- tests/service/Symbols.fs | 10 +- .../src/FSharp.Editor/CodeFix/SimplifyName.fs | 3 - .../src/FSharp.Editor/Common/Extensions.fs | 4 +- .../SimplifyNameDiagnosticAnalyzer.fs | 7 +- .../Diagnostics/UnusedDeclarationsAnalyzer.fs | 2 +- .../UnusedOpensDiagnosticAnalyzer.fs | 7 - 18 files changed, 772 insertions(+), 169 deletions(-) create mode 100644 tests/benchmarks/CompilerServiceBenchmarks/decentlySizedStandAloneFile.fsx diff --git a/src/fsharp/service/FSharpCheckerResults.fs b/src/fsharp/service/FSharpCheckerResults.fs index 9de684583c1..c1e0b96d002 100644 --- a/src/fsharp/service/FSharpCheckerResults.fs +++ b/src/fsharp/service/FSharpCheckerResults.fs @@ -1872,16 +1872,18 @@ type FSharpCheckFileResults member __.DependencyFiles = dependencyFiles member __.GetAllUsesOfAllSymbolsInFile(?cancellationToken: CancellationToken ) = - threadSafeOp - (fun () -> [| |]) + threadSafeOp + (fun () -> Seq.empty) (fun scope -> let cenv = scope.SymbolEnv - [| for symbolUseChunk in scope.ScopeSymbolUses.AllUsesOfSymbols do - for symbolUse in symbolUseChunk do - cancellationToken |> Option.iter (fun ct -> ct.ThrowIfCancellationRequested()) - if symbolUse.ItemOccurence <> ItemOccurence.RelatedText then - let symbol = FSharpSymbol.Create(cenv, symbolUse.Item) - yield FSharpSymbolUse(scope.TcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range) |]) + seq { + for symbolUseChunk in scope.ScopeSymbolUses.AllUsesOfSymbols do + for symbolUse in symbolUseChunk do + cancellationToken |> Option.iter (fun ct -> ct.ThrowIfCancellationRequested()) + if symbolUse.ItemOccurence <> ItemOccurence.RelatedText then + let symbol = FSharpSymbol.Create(cenv, symbolUse.Item) + FSharpSymbolUse(scope.TcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range) + }) member __.GetUsesOfSymbolInFile(symbol:FSharpSymbol, ?cancellationToken: CancellationToken) = threadSafeOp diff --git a/src/fsharp/service/FSharpCheckerResults.fsi b/src/fsharp/service/FSharpCheckerResults.fsi index 023c23d3ac7..95e8156b748 100644 --- a/src/fsharp/service/FSharpCheckerResults.fsi +++ b/src/fsharp/service/FSharpCheckerResults.fsi @@ -210,7 +210,7 @@ type public FSharpCheckFileResults = member GetFormatSpecifierLocationsAndArity : unit -> (range*int)[] /// Get all textual usages of all symbols throughout the file - member GetAllUsesOfAllSymbolsInFile : ?cancellationToken: CancellationToken -> FSharpSymbolUse[] + member GetAllUsesOfAllSymbolsInFile : ?cancellationToken: CancellationToken -> seq /// Get the textual usages that resolved to the given symbol throughout the file member GetUsesOfSymbolInFile : symbol:FSharpSymbol * ?cancellationToken: CancellationToken -> FSharpSymbolUse[] diff --git a/src/fsharp/service/ServiceAnalysis.fs b/src/fsharp/service/ServiceAnalysis.fs index 3eccf417a7b..f549065fc1c 100644 --- a/src/fsharp/service/ServiceAnalysis.fs +++ b/src/fsharp/service/ServiceAnalysis.fs @@ -2,6 +2,8 @@ namespace FSharp.Compiler.SourceCodeServices +open System.Threading + open FSharp.Compiler open FSharp.Compiler.Range open FSharp.Compiler.PrettyNaming @@ -54,14 +56,15 @@ module UnusedOpens = member __.RevealedSymbolsContains(symbol) = revealedSymbols.Force().Contains symbol type OpenedModuleGroup = - { OpenedModules: OpenedModule list } + { OpenedModules: OpenedModule [] } static member Create (modul: FSharpEntity) = let rec getModuleAndItsAutoOpens (isNestedAutoOpen: bool) (modul: FSharpEntity) = - [ yield OpenedModule (modul, isNestedAutoOpen) - for ent in modul.NestedEntities do - if ent.IsFSharpModule && Symbol.hasAttribute ent.Attributes then - yield! getModuleAndItsAutoOpens true ent ] + [| + yield OpenedModule (modul, isNestedAutoOpen) + for ent in modul.NestedEntities do + if ent.IsFSharpModule && Symbol.hasAttribute ent.Attributes then + yield! getModuleAndItsAutoOpens true ent |] { OpenedModules = getModuleAndItsAutoOpens false modul } /// Represents single open statement. @@ -78,59 +81,62 @@ module UnusedOpens = /// Gets the open statements, their scopes and their resolutions let getOpenStatements (openDeclarations: FSharpOpenDeclaration[]) : OpenStatement[] = openDeclarations - |> Array.filter (fun x -> not x.IsOwnNamespace) |> Array.choose (fun openDecl -> - match openDecl.LongId, openDecl.Range with - | firstId :: _, Some range -> - if firstId.idText = MangledGlobalName then - None - else - Some { OpenedGroups = openDecl.Modules |> List.map OpenedModuleGroup.Create - Range = range - AppliedScope = openDecl.AppliedScope } - | _ -> None) + if openDecl.IsOwnNamespace then + None + else + match openDecl.LongId, openDecl.Range with + | firstId :: _, Some range -> + if firstId.idText = MangledGlobalName then + None + else + Some { OpenedGroups = openDecl.Modules |> List.map OpenedModuleGroup.Create + Range = range + AppliedScope = openDecl.AppliedScope } + | _ -> None) /// Only consider symbol uses which are the first part of a long ident, i.e. with no qualifying identifiers - let filterSymbolUses (getSourceLineStr: int -> string) (symbolUses: FSharpSymbolUse[]) : FSharpSymbolUse[] = + let filterSymbolUses (getSourceLineStr: int -> string) (symbolUses: seq) = symbolUses - |> Array.filter (fun su -> - match su.Symbol with - | :? FSharpMemberOrFunctionOrValue as fv when fv.IsExtensionMember -> + |> Seq.filter(fun (su: FSharpSymbolUse) -> + match su.Symbol with + | :? FSharpMemberOrFunctionOrValue as fv when fv.IsExtensionMember -> // Extension members should be taken into account even though they have a prefix (as they do most of the time) true - | :? FSharpMemberOrFunctionOrValue as fv when not fv.IsModuleValueOrMember -> + | :? FSharpMemberOrFunctionOrValue as fv when not fv.IsModuleValueOrMember -> // Local values can be ignored false - | :? FSharpMemberOrFunctionOrValue when su.IsFromDefinition -> + | :? FSharpMemberOrFunctionOrValue when su.IsFromDefinition -> // Value definitions should be ignored false - | :? FSharpGenericParameter -> + | :? FSharpGenericParameter -> // Generic parameters can be ignored, they never come into scope via 'open' false - | :? FSharpUnionCase when su.IsFromDefinition -> + | :? FSharpUnionCase when su.IsFromDefinition -> false - | :? FSharpField as field when - field.DeclaringEntity.IsSome && field.DeclaringEntity.Value.IsFSharpRecord -> + | :? FSharpField as field when + field.DeclaringEntity.IsSome && field.DeclaringEntity.Value.IsFSharpRecord -> // Record fields are used in name resolution true - | :? FSharpField as field when field.IsUnionCaseField -> - false + | :? FSharpField as field when field.IsUnionCaseField -> + false - | _ -> + | _ -> // For the rest of symbols we pick only those which are the first part of a long ident, because it's they which are // contained in opened namespaces / modules. For example, we pick `IO` from long ident `IO.File.OpenWrite` because // it's `open System` which really brings it into scope. let partialName = QuickParse.GetPartialLongNameEx (getSourceLineStr su.RangeAlternate.StartLine, su.RangeAlternate.EndColumn - 1) List.isEmpty partialName.QualifyingIdents) + |> Array.ofSeq /// Split symbol uses into cases that are easy to handle (via DeclaringEntity) and those that don't have a good DeclaringEntity - let splitSymbolUses (symbolUses: FSharpSymbolUse[]) : FSharpSymbolUse[] * FSharpSymbolUse[] = + let splitSymbolUses (symbolUses: FSharpSymbolUse[]) = symbolUses |> Array.partition (fun symbolUse -> let symbol = symbolUse.Symbol match symbol with @@ -152,18 +158,18 @@ module UnusedOpens = openStatement.OpenedGroups |> List.choose (fun openedGroup -> let openedEntitiesToExamine = openedGroup.OpenedModules - |> List.filter (fun openedEntity -> + |> Array.filter (fun openedEntity -> not (usedModules.BagExistsValueForKey(openedEntity.Entity, fun scope -> rangeContainsRange scope openStatement.AppliedScope))) match openedEntitiesToExamine with - | [] -> None - | _ when openedEntitiesToExamine |> List.exists (fun x -> not x.IsNestedAutoOpen) -> Some { OpenedModules = openedEntitiesToExamine } + | [||] -> None + | _ when openedEntitiesToExamine |> Array.exists (fun x -> not x.IsNestedAutoOpen) -> Some { OpenedModules = openedEntitiesToExamine } | _ -> None) // Find the opened groups that are used by some symbol use let newlyUsedOpenedGroups = openedGroupsToExamine |> List.filter (fun openedGroup -> - openedGroup.OpenedModules |> List.exists (fun openedEntity -> + openedGroup.OpenedModules |> Array.exists (fun openedEntity -> symbolUsesRangesByDeclaringEntity.BagExistsValueForKey(openedEntity.Entity, fun symbolUseRange -> rangeContainsRange openStatement.AppliedScope symbolUseRange && Range.posGt symbolUseRange.Start openStatement.Range.End) || @@ -174,14 +180,14 @@ module UnusedOpens = openedEntity.RevealedSymbolsContains symbolUse.Symbol))) // Return them as interim used entities - let newlyOpenedModules = newlyUsedOpenedGroups |> List.collect (fun openedGroup -> openedGroup.OpenedModules) + let newlyOpenedModules = newlyUsedOpenedGroups |> List.collect (fun openedGroup -> openedGroup.OpenedModules |> List.ofArray) for openedModule in newlyOpenedModules do let scopes = match usedModules.TryGetValue openedModule.Entity with | true, scopes -> openStatement.AppliedScope :: scopes | _ -> [openStatement.AppliedScope] usedModules.[openedModule.Entity] <- scopes - not (isNil newlyOpenedModules) + not (newlyOpenedModules.IsEmpty) /// Incrementally filter out the open statements one by one. Filter those whose contents are referred to somewhere in the symbol uses. /// Async to allow cancellation. @@ -241,63 +247,64 @@ module SimplifyNames = let getPlidLength (plid: string list) = (plid |> List.sumBy String.length) + plid.Length - let getSimplifiableNames (checkFileResults: FSharpCheckFileResults, getSourceLineStr: int -> string) : Async = + let getSimplifiableNames (checkFileResults: FSharpCheckFileResults, getSourceLineStr: int -> string) = async { let result = ResizeArray() let! ct = Async.CancellationToken - let symbolUses = checkFileResults.GetAllUsesOfAllSymbolsInFile(ct) let symbolUses = - symbolUses - |> Array.filter (fun symbolUse -> not symbolUse.IsFromOpenStatement) - |> Array.Parallel.map (fun symbolUse -> - let lineStr = getSourceLineStr symbolUse.RangeAlternate.StartLine - // for `System.DateTime.Now` it returns ([|"System"; "DateTime"|], "Now") - let partialName = QuickParse.GetPartialLongNameEx(lineStr, symbolUse.RangeAlternate.EndColumn - 1) - // `symbolUse.RangeAlternate.Start` does not point to the start of plid, it points to start of `name`, - // so we have to calculate plid's start ourselves. - let plidStartCol = symbolUse.RangeAlternate.EndColumn - partialName.PartialIdent.Length - (getPlidLength partialName.QualifyingIdents) - symbolUse, partialName.QualifyingIdents, plidStartCol, partialName.PartialIdent) - |> Array.filter (fun (_, plid, _, name) -> name <> "" && not (List.isEmpty plid)) - |> Array.groupBy (fun (symbolUse, _, plidStartCol, _) -> symbolUse.RangeAlternate.StartLine, plidStartCol) - |> Array.map (fun (_, xs) -> xs |> Array.maxBy (fun (symbolUse, _, _, _) -> symbolUse.RangeAlternate.EndColumn)) + checkFileResults.GetAllUsesOfAllSymbolsInFile(ct) + |> Seq.choose (fun symbolUse -> + if symbolUse.IsFromOpenStatement || symbolUse.IsFromDefinition then + None + else + let lineStr = getSourceLineStr symbolUse.RangeAlternate.StartLine + // for `System.DateTime.Now` it returns ([|"System"; "DateTime"|], "Now") + let partialName = QuickParse.GetPartialLongNameEx(lineStr, symbolUse.RangeAlternate.EndColumn - 1) + // `symbolUse.RangeAlternate.Start` does not point to the start of plid, it points to start of `name`, + // so we have to calculate plid's start ourselves. + let plidStartCol = symbolUse.RangeAlternate.EndColumn - partialName.PartialIdent.Length - (getPlidLength partialName.QualifyingIdents) + if partialName.PartialIdent = "" || List.isEmpty partialName.QualifyingIdents then + None + else + Some (symbolUse, partialName.QualifyingIdents, plidStartCol, partialName.PartialIdent)) + |> Seq.groupBy (fun (symbolUse, _, plidStartCol, _) -> symbolUse.RangeAlternate.StartLine, plidStartCol) + |> Seq.map (fun (_, xs) -> xs |> Seq.maxBy (fun (symbolUse, _, _, _) -> symbolUse.RangeAlternate.EndColumn)) for symbolUse, plid, plidStartCol, name in symbolUses do - if not symbolUse.IsFromDefinition then - let posAtStartOfName = - let r = symbolUse.RangeAlternate - if r.StartLine = r.EndLine then Range.mkPos r.StartLine (r.EndColumn - name.Length) - else r.Start - - let getNecessaryPlid (plid: string list) : string list = - let rec loop (rest: string list) (current: string list) = - match rest with - | [] -> current - | headIdent :: restPlid -> - let res = checkFileResults.IsRelativeNameResolvableFromSymbol(posAtStartOfName, current, symbolUse.Symbol) - if res then current - else loop restPlid (headIdent :: current) - loop (List.rev plid) [] + let posAtStartOfName = + let r = symbolUse.RangeAlternate + if r.StartLine = r.EndLine then Range.mkPos r.StartLine (r.EndColumn - name.Length) + else r.Start + + let getNecessaryPlid (plid: string list) : string list = + let rec loop (rest: string list) (current: string list) = + match rest with + | [] -> current + | headIdent :: restPlid -> + let res = checkFileResults.IsRelativeNameResolvableFromSymbol(posAtStartOfName, current, symbolUse.Symbol) + if res then current + else loop restPlid (headIdent :: current) + loop (List.rev plid) [] - let necessaryPlid = getNecessaryPlid plid + let necessaryPlid = getNecessaryPlid plid - match necessaryPlid with - | necessaryPlid when necessaryPlid = plid -> () - | necessaryPlid -> - let r = symbolUse.RangeAlternate - let necessaryPlidStartCol = r.EndColumn - name.Length - (getPlidLength necessaryPlid) + match necessaryPlid with + | necessaryPlid when necessaryPlid = plid -> () + | necessaryPlid -> + let r = symbolUse.RangeAlternate + let necessaryPlidStartCol = r.EndColumn - name.Length - (getPlidLength necessaryPlid) - let unnecessaryRange = - Range.mkRange r.FileName (Range.mkPos r.StartLine plidStartCol) (Range.mkPos r.EndLine necessaryPlidStartCol) + let unnecessaryRange = + Range.mkRange r.FileName (Range.mkPos r.StartLine plidStartCol) (Range.mkPos r.EndLine necessaryPlidStartCol) - let relativeName = (String.concat "." plid) + "." + name - result.Add({Range = unnecessaryRange; RelativeName = relativeName}) + let relativeName = (String.concat "." plid) + "." + name + result.Add({Range = unnecessaryRange; RelativeName = relativeName}) - return List.ofSeq result + return (result :> seq<_>) } module UnusedDeclarations = let isPotentiallyUnusedDeclaration (symbol: FSharpSymbol) : bool = - match symbol with // Determining that a record, DU or module is used anywhere requires inspecting all their enclosed entities (fields, cases and func / vals) @@ -314,33 +321,29 @@ module UnusedDeclarations = | :? FSharpParameter -> false | _ -> true - let getUnusedDeclarationRanges (symbolsUses: FSharpSymbolUse[]) (isScript: bool) = - let definitions = - symbolsUses - |> Array.filter (fun su -> - su.IsFromDefinition && - su.Symbol.DeclarationLocation.IsSome && - (isScript || su.IsPrivateToFile) && - not (su.Symbol.DisplayName.StartsWith "_") && - isPotentiallyUnusedDeclaration su.Symbol) - + let getUnusedDeclarationRanges (symbolsUses: seq) (isScript: bool) = let usages = let usages = symbolsUses - |> Array.filter (fun su -> not su.IsFromDefinition) - |> Array.choose (fun su -> su.Symbol.DeclarationLocation) + |> Seq.choose (fun su -> if not su.IsFromDefinition then su.Symbol.DeclarationLocation else None) HashSet(usages) - let unusedRanges = - definitions - |> Array.map (fun defSu -> defSu, usages.Contains defSu.Symbol.DeclarationLocation.Value) - |> Array.groupBy (fun (defSu, _) -> defSu.RangeAlternate) - |> Array.filter (fun (_, defSus) -> defSus |> Array.forall (fun (_, isUsed) -> not isUsed)) - |> Array.map (fun (m, _) -> m) - - Array.toList unusedRanges + symbolsUses + |> Seq.choose(fun (su: FSharpSymbolUse) -> + if su.IsFromDefinition && + su.Symbol.DeclarationLocation.IsSome && + (isScript || su.IsPrivateToFile) && + not (su.Symbol.DisplayName.StartsWith "_") && + isPotentiallyUnusedDeclaration su.Symbol + then + Some (su, usages.Contains su.Symbol.DeclarationLocation.Value) + else + None) + |> Seq.groupBy (fun (defSu, _) -> defSu.RangeAlternate) + |> Seq.filter (fun (_, defSus) -> defSus |> Seq.forall (fun (_, isUsed) -> not isUsed)) + |> Seq.map (fun (m, _) -> m) - let getUnusedDeclarations(checkFileResults: FSharpCheckFileResults, isScriptFile: bool) : Async = + let getUnusedDeclarations(checkFileResults: FSharpCheckFileResults, isScriptFile: bool) = async { let! ct = Async.CancellationToken let allSymbolUsesInFile = checkFileResults.GetAllUsesOfAllSymbolsInFile(ct) diff --git a/src/fsharp/service/ServiceAnalysis.fsi b/src/fsharp/service/ServiceAnalysis.fsi index 9d09c5e3e90..3fa2be9ea89 100644 --- a/src/fsharp/service/ServiceAnalysis.fsi +++ b/src/fsharp/service/ServiceAnalysis.fsi @@ -2,9 +2,7 @@ namespace FSharp.Compiler.SourceCodeServices -open FSharp.Compiler.NameResolution open FSharp.Compiler.Range -open FSharp.Compiler.SyntaxTree module public UnusedOpens = @@ -24,9 +22,9 @@ module public SimplifyNames = } /// Get all ranges that can be simplified in a file - val getSimplifiableNames : checkFileResults: FSharpCheckFileResults * getSourceLineStr: (int -> string) -> Async + val getSimplifiableNames : checkFileResults: FSharpCheckFileResults * getSourceLineStr: (int -> string) -> Async> module public UnusedDeclarations = /// Get all unused declarations in a file - val getUnusedDeclarations : checkFileResults: FSharpCheckFileResults * isScriptFile: bool -> Async + val getUnusedDeclarations : checkFileResults: FSharpCheckFileResults * isScriptFile: bool -> Async> diff --git a/tests/FSharp.Compiler.Service.Tests/SurfaceArea.netstandard.fs b/tests/FSharp.Compiler.Service.Tests/SurfaceArea.netstandard.fs index 5bebc9201d0..1f94595981f 100644 --- a/tests/FSharp.Compiler.Service.Tests/SurfaceArea.netstandard.fs +++ b/tests/FSharp.Compiler.Service.Tests/SurfaceArea.netstandard.fs @@ -21450,7 +21450,7 @@ FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.Sourc FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.SourceCodeServices.FSharpDeclarationListInfo GetDeclarationListInfo(Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.SourceCodeServices.FSharpParseFileResults], Int32, System.String, FSharp.Compiler.PartialLongName, Microsoft.FSharp.Core.FSharpOption`1[Microsoft.FSharp.Core.FSharpFunc`2[Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SourceCodeServices.AssemblySymbol]]]) FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.SourceCodeServices.FSharpFindDeclResult GetDeclarationLocation(Int32, Int32, System.String, Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean]) FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.SourceCodeServices.FSharpMethodGroup GetMethods(Int32, Int32, System.String, Microsoft.FSharp.Core.FSharpOption`1[Microsoft.FSharp.Collections.FSharpList`1[System.String]]) -FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.SourceCodeServices.FSharpSymbolUse[] GetAllUsesOfAllSymbolsInFile(Microsoft.FSharp.Core.FSharpOption`1[System.Threading.CancellationToken]) +FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: System.Collections.Generic.IEnumerable`1[FSharp.Compiler.SourceCodeServices.FSharpSymbolUse] GetAllUsesOfAllSymbolsInFile(Microsoft.FSharp.Core.FSharpOption`1[System.Threading.CancellationToken]) FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.SourceCodeServices.FSharpSymbolUse[] GetUsesOfSymbolInFile(FSharp.Compiler.SourceCodeServices.FSharpSymbol, Microsoft.FSharp.Core.FSharpOption`1[System.Threading.CancellationToken]) FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.SourceCodeServices.FSharpToolTipText`1[Internal.Utilities.StructuredFormat.Layout] GetStructuredToolTipText(Int32, Int32, System.String, Microsoft.FSharp.Collections.FSharpList`1[System.String], Int32) FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults: FSharp.Compiler.SourceCodeServices.FSharpToolTipText`1[System.String] GetToolTipText(Int32, Int32, System.String, Microsoft.FSharp.Collections.FSharpList`1[System.String], Int32) @@ -24992,7 +24992,7 @@ FSharp.Compiler.SourceCodeServices.SimplifyNames+SimplifiableRange: Void .ctor(r FSharp.Compiler.SourceCodeServices.SimplifyNames+SimplifiableRange: range Range FSharp.Compiler.SourceCodeServices.SimplifyNames+SimplifiableRange: range get_Range() FSharp.Compiler.SourceCodeServices.SimplifyNames: FSharp.Compiler.SourceCodeServices.SimplifyNames+SimplifiableRange -FSharp.Compiler.SourceCodeServices.SimplifyNames: Microsoft.FSharp.Control.FSharpAsync`1[Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SourceCodeServices.SimplifyNames+SimplifiableRange]] getSimplifiableNames(FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults, Microsoft.FSharp.Core.FSharpFunc`2[System.Int32,System.String]) +FSharp.Compiler.SourceCodeServices.SimplifyNames: Microsoft.FSharp.Control.FSharpAsync`1[System.Collections.Generic.IEnumerable`1[FSharp.Compiler.SourceCodeServices.SimplifyNames+SimplifiableRange]] getSimplifiableNames(FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults, Microsoft.FSharp.Core.FSharpFunc`2[System.Int32,System.String]) FSharp.Compiler.SourceCodeServices.SourceFile: Boolean IsCompilable(System.String) FSharp.Compiler.SourceCodeServices.SourceFile: Boolean MustBeSingleFileProject(System.String) FSharp.Compiler.SourceCodeServices.Structure+Collapse+Tags: Int32 Below @@ -25356,7 +25356,7 @@ FSharp.Compiler.SourceCodeServices.UntypedParseImpl: Microsoft.FSharp.Core.FShar FSharp.Compiler.SourceCodeServices.UntypedParseImpl: Microsoft.FSharp.Core.FSharpOption`1[System.String] TryFindExpressionIslandInPosition(pos, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.SyntaxTree+ParsedInput]) FSharp.Compiler.SourceCodeServices.UntypedParseImpl: Microsoft.FSharp.Core.FSharpOption`1[System.Tuple`2[FSharp.Compiler.Range+pos,System.Boolean]] TryFindExpressionASTLeftOfDotLeftOfCursor(pos, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.SyntaxTree+ParsedInput]) FSharp.Compiler.SourceCodeServices.UntypedParseImpl: System.String[] GetFullNameOfSmallestModuleOrNamespaceAtPoint(ParsedInput, pos) -FSharp.Compiler.SourceCodeServices.UnusedDeclarations: Microsoft.FSharp.Control.FSharpAsync`1[Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Range+range]] getUnusedDeclarations(FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults, Boolean) +FSharp.Compiler.SourceCodeServices.UnusedDeclarations: Microsoft.FSharp.Control.FSharpAsync`1[System.Collections.Generic.IEnumerable`1[FSharp.Compiler.Range+range]] getUnusedDeclarations(FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults, Boolean) FSharp.Compiler.SourceCodeServices.UnusedOpens: Microsoft.FSharp.Control.FSharpAsync`1[Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Range+range]] getUnusedOpens(FSharp.Compiler.SourceCodeServices.FSharpCheckFileResults, Microsoft.FSharp.Core.FSharpFunc`2[System.Int32,System.String]) FSharp.Compiler.SourceCodeServices.XmlDocComment: Microsoft.FSharp.Core.FSharpOption`1[System.Int32] isBlank(System.String) FSharp.Compiler.SourceCodeServices.XmlDocParser: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.SourceCodeServices.XmlDocable] getXmlDocables(FSharp.Compiler.Text.ISourceText, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.SyntaxTree+ParsedInput]) diff --git a/tests/benchmarks/Benchmarks.sln b/tests/benchmarks/Benchmarks.sln index 3d490e60b8c..b2a35f33b01 100644 --- a/tests/benchmarks/Benchmarks.sln +++ b/tests/benchmarks/Benchmarks.sln @@ -5,9 +5,9 @@ VisualStudioVersion = 16.0.28407.52 MinimumVisualStudioVersion = 10.0.40219.1 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "CompilerServiceBenchmarks", "CompilerServiceBenchmarks\CompilerServiceBenchmarks.fsproj", "{9A3C565C-B514-4AE0-8B01-CA80E8453EB0}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Core", "..\src\fsharp\FSharp.Core\FSharp.Core.fsproj", "{DED3BBD7-53F4-428A-8C9F-27968E768605}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Compiler.Service", "..\..\src\fsharp\FSharp.Compiler.Service\FSharp.Compiler.Service.fsproj", "{0BD3108C-8CDC-48E3-8CD3-B50E4F2E72A4}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Compiler.Private", "..\src\fsharp\FSharp.Compiler.Private\FSharp.Compiler.Private.fsproj", "{2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Core", "..\..\src\fsharp\FSharp.Core\FSharp.Core.fsproj", "{A2E3D114-10EE-4438-9C1D-2E15046607F3}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -22,18 +22,18 @@ Global {9A3C565C-B514-4AE0-8B01-CA80E8453EB0}.Proto|Any CPU.Build.0 = Release|Any CPU {9A3C565C-B514-4AE0-8B01-CA80E8453EB0}.Release|Any CPU.ActiveCfg = Release|Any CPU {9A3C565C-B514-4AE0-8B01-CA80E8453EB0}.Release|Any CPU.Build.0 = Release|Any CPU - {DED3BBD7-53F4-428A-8C9F-27968E768605}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DED3BBD7-53F4-428A-8C9F-27968E768605}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DED3BBD7-53F4-428A-8C9F-27968E768605}.Proto|Any CPU.ActiveCfg = Debug|Any CPU - {DED3BBD7-53F4-428A-8C9F-27968E768605}.Proto|Any CPU.Build.0 = Debug|Any CPU - {DED3BBD7-53F4-428A-8C9F-27968E768605}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DED3BBD7-53F4-428A-8C9F-27968E768605}.Release|Any CPU.Build.0 = Release|Any CPU - {2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}.Proto|Any CPU.ActiveCfg = Debug|Any CPU - {2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}.Proto|Any CPU.Build.0 = Debug|Any CPU - {2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}.Release|Any CPU.Build.0 = Release|Any CPU + {0BD3108C-8CDC-48E3-8CD3-B50E4F2E72A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0BD3108C-8CDC-48E3-8CD3-B50E4F2E72A4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0BD3108C-8CDC-48E3-8CD3-B50E4F2E72A4}.Proto|Any CPU.ActiveCfg = Debug|Any CPU + {0BD3108C-8CDC-48E3-8CD3-B50E4F2E72A4}.Proto|Any CPU.Build.0 = Debug|Any CPU + {0BD3108C-8CDC-48E3-8CD3-B50E4F2E72A4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0BD3108C-8CDC-48E3-8CD3-B50E4F2E72A4}.Release|Any CPU.Build.0 = Release|Any CPU + {A2E3D114-10EE-4438-9C1D-2E15046607F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A2E3D114-10EE-4438-9C1D-2E15046607F3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A2E3D114-10EE-4438-9C1D-2E15046607F3}.Proto|Any CPU.ActiveCfg = Debug|Any CPU + {A2E3D114-10EE-4438-9C1D-2E15046607F3}.Proto|Any CPU.Build.0 = Debug|Any CPU + {A2E3D114-10EE-4438-9C1D-2E15046607F3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A2E3D114-10EE-4438-9C1D-2E15046607F3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tests/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj b/tests/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj index 2fb509e4c14..5ee4d5f5ad2 100644 --- a/tests/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj +++ b/tests/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj @@ -17,17 +17,20 @@ + + + - - + + diff --git a/tests/benchmarks/CompilerServiceBenchmarks/Program.fs b/tests/benchmarks/CompilerServiceBenchmarks/Program.fs index 313e45cf902..74ccbe06f5e 100644 --- a/tests/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/tests/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -4,6 +4,7 @@ open System.Text open FSharp.Compiler.ErrorLogger open FSharp.Compiler.SourceCodeServices open FSharp.Compiler.Text +open FSharp.Compiler.Range open FSharp.Compiler.AbstractIL open FSharp.Compiler.AbstractIL.IL open FSharp.Compiler.AbstractIL.ILBinaryReader @@ -123,12 +124,14 @@ let function%s (x: %s) = let z = x + y z""" moduleName moduleName moduleName moduleName + let decentlySizedStandAloneFile = File.ReadAllText(Path.Combine(__SOURCE_DIRECTORY__, "decentlySizedStandAloneFile.fsx")) + [] type CompilerService() = - let mutable checkerOpt = None - let mutable sourceOpt = None + let mutable assembliesOpt = None + let mutable decentlySizedStandAloneFileCheckResultOpt = None let parsingOptions = { @@ -141,8 +144,6 @@ type CompilerService() = IsExe = false } - let mutable assembliesOpt = None - let readerOptions = { pdbDirPath = None @@ -168,6 +169,18 @@ type CompilerService() = System.AppDomain.CurrentDomain.GetAssemblies() |> Array.map (fun x -> (x.Location)) |> Some + + | _ -> () + + match decentlySizedStandAloneFileCheckResultOpt with + | None -> + let options, _ = + checkerOpt.Value.GetProjectOptionsFromScript("decentlySizedStandAloneFile.fsx", SourceText.ofString decentlySizedStandAloneFile) + |> Async.RunSynchronously + let _, checkResult = + checkerOpt.Value.ParseAndCheckFileInProject("decentlySizedStandAloneFile.fsx", 0, SourceText.ofString decentlySizedStandAloneFile, options) + |> Async.RunSynchronously + decentlySizedStandAloneFileCheckResultOpt <- Some checkResult | _ -> () [] @@ -292,6 +305,8 @@ type CompilerService() = // If set to 3, it will be almost as slow as re-evaluating all project and it's projects references on setup; this could be a bug or not what we want. this.TypeCheckFileWith100ReferencedProjectsRun() + member val TypeCheckFileWithNoReferencesOptions = createProject "MainProject" [] + [] member this.TypeCheckFileWith100ReferencedProjectsCleanup() = this.TypeCheckFileWith100ReferencedProjectsOptions.SourceFiles @@ -314,7 +329,45 @@ type CompilerService() = checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() ClearAllILModuleReaderCache() + [] + member this.SimplifyNames() = + match decentlySizedStandAloneFileCheckResultOpt with + | Some checkResult -> + match checkResult with + | FSharpCheckFileAnswer.Aborted -> failwith "checker aborted" + | FSharpCheckFileAnswer.Succeeded results -> + let sourceLines = decentlySizedStandAloneFile.Split ([|"\r\n"; "\n"; "\r"|], StringSplitOptions.None) + let ranges = SimplifyNames.getSimplifiableNames(results, fun lineNum -> sourceLines.[Line.toZ lineNum]) |> Async.RunSynchronously + ignore ranges + () + | _ -> failwith "oopsie" + + [] + member this.UnusedOpens() = + match decentlySizedStandAloneFileCheckResultOpt with + | Some checkResult -> + match checkResult with + | FSharpCheckFileAnswer.Aborted -> failwith "checker aborted" + | FSharpCheckFileAnswer.Succeeded results -> + let sourceLines = decentlySizedStandAloneFile.Split ([|"\r\n"; "\n"; "\r"|], StringSplitOptions.None) + let decls = UnusedOpens.getUnusedOpens(results, fun lineNum -> sourceLines.[Line.toZ lineNum]) |> Async.RunSynchronously + ignore decls + () + | _ -> failwith "oopsie" + + [] + member this.UnusedDeclarations() = + match decentlySizedStandAloneFileCheckResultOpt with + | Some checkResult -> + match checkResult with + | FSharpCheckFileAnswer.Aborted -> failwith "checker aborted" + | FSharpCheckFileAnswer.Succeeded results -> + let decls = UnusedDeclarations.getUnusedDeclarations(results, true) |> Async.RunSynchronously + ignore decls // should be 16 + () + | _ -> failwith "oopsie" + [] -let main argv = - let _ = BenchmarkRunner.Run() +let main _ = + BenchmarkRunner.Run() |> ignore 0 diff --git a/tests/benchmarks/CompilerServiceBenchmarks/decentlySizedStandAloneFile.fsx b/tests/benchmarks/CompilerServiceBenchmarks/decentlySizedStandAloneFile.fsx new file mode 100644 index 00000000000..95b2f151be6 --- /dev/null +++ b/tests/benchmarks/CompilerServiceBenchmarks/decentlySizedStandAloneFile.fsx @@ -0,0 +1,547 @@ +module Parser = + + (* + F# implementation of a generic Top-Down-Operator-Precedence Parser + as described in this paper http://portal.acm.org/citation.cfm?id=512931. + + The parser has been extended to allow for statements in comparison to Pratt's + original algorithm which only parsed languages which use expression-only grammar. + + The parsers is "impure" in the sense that is uses a ref-cell for storing the + input in the T<_, _, _> record class, this is soley for performance reasons + as it's to expensive to create a new record object for every consumed token. + Certain functions also throw exceptions, which generally also is considered impure. + + The parser produces nice error message in this style: + + Error on line: 5 col: 9 + 4: if(x == y) { + 5: print'x equals y'); + ----------^ + Unexpected: #string "x equals y" + + More information: + * http://en.wikipedia.org/wiki/Vaughan_Pratt (Original Inventor) + * http://en.wikipedia.org/wiki/Pratt_parser (Alias name) + * http://effbot.org/zone/simple-top-down-parsing.htm (Python implementation) + * http://javascript.crockford.com/tdop/tdop.html (JavaScript implementation) + *) + + type Pos = int * int + + type T<'a, 'b, 'c> when 'c : comparison = { + Input : 'a list ref + Lines : string option + + Type : 'a -> 'c + Position : 'a -> Pos + PrettyPrint : ('a -> string) option + + //Parser definitions and binding powers + BindingPower : Map<'c, int> + Null : Map<'c, 'a -> T<'a, 'b, 'c> -> 'b> + Stmt : Map<'c, 'a -> T<'a, 'b, 'c> -> 'b> + Left : Map<'c, 'a -> 'b -> T<'a, 'b, 'c> -> 'b> + } + + type Pattern<'a, 'b, 'c> when 'c : comparison + = Sym of 'c + | Get of (T<'a, 'b, 'c> -> 'b) + + //Errors + type Exn (msg, pos) = + inherit System.Exception(msg) + member x.Position = pos + + (* + Creates a string error snippet + that points out the exact source position + where the error occured, for example: + + 4: if(x == y) { + 5: print'x equals y'); + ----------^ + *) + let errorSource pos source = + + let splitLines (text:string) = + let text = text.Replace("\r\n", "\n").Replace("\r", "\n") + System.Text.RegularExpressions.Regex.Split(text, "\n") + + let lineNum (input:int) n = + (input.ToString()).PadLeft(n, '0') + + let stringRepeat n input = + if System.String.IsNullOrEmpty input then input + else + let result = new System.Text.StringBuilder(input.Length * n) + result.Insert(0, input, n).ToString() + + match source with + | None -> "" + | Some(source:string) -> + let source = source |> splitLines + let result = ref "" + let line, column = pos + + if line <= source.Length && line > 1 then + let nr = line.ToString() + let nrl = nr.Length + + //previous line + let pline = line - 1 + if pline >= 1 then + let num = lineNum pline nrl + result := num+": "+source.[pline-1]+"\n" + + //current line + let text = source.[line-1] + if column <= text.Length then + let arrow = "-" |> stringRepeat (nrl + column) + result := !result+nr+": "+text+"\n"+arrow+"^\n" + + !result + + let exn msg = Exn(msg, (0, 0)) |> raise + let exnLine pos msg = + let line = sprintf "Error on line: %i col: %i\n" (fst pos) (snd pos) + Exn(line + msg, pos) |> raise + + let private unexpectedEnd () = "Unexpected end of input" |> exn + let private unexpectedToken token parser = + let type' = + match parser.PrettyPrint with + | None -> (token |> parser.Type).ToString() + | Some f -> token |> f + + let pos = token |> parser.Position + let source = parser.Lines |> errorSource pos + let expected = sprintf "Unexpected: %s" type' + (source + expected) |> exnLine pos + + let inline private getBindingPower tok parser = + let pwr = parser.BindingPower.TryFind (parser.Type tok) + match pwr with Some pwr -> pwr | _ -> 0 + + let current parser = + match !parser.Input with + | token::_ -> token + | _ -> unexpectedEnd () + + let currentTry parser = + match !parser.Input with + | token::_ -> Some token + | _ -> None + + let currentType parser = + parser |> current |> parser.Type + + let currentTypeTry parser = + match parser |> currentTry with + | Some token -> Some(token |> parser.Type) + | _ -> None + + let skip parser = + match !parser.Input with + | _::input -> parser.Input := input + | _ -> unexpectedEnd () + + let skipIf type' parser = + match !parser.Input with + | token::xs when parser.Type token = type' -> + parser.Input := xs + + | token::_ -> + unexpectedToken token parser + + | _ -> unexpectedEnd () + + let skipCurrent parser = + let current = parser |> current + parser |> skip + current + + let exprPwr rbpw parser = + let rec expr left = + match parser |> currentTry with + | Some token when rbpw < (parser |> getBindingPower token) -> + parser |> skip + + let type' = parser.Type token + let led = + match parser.Left.TryFind type' with + | None -> unexpectedToken token parser + | Some led -> led + + led token left parser |> expr + + | _ -> left + + let tok = parser |> skipCurrent + let type' = parser.Type tok + let nud = + match parser.Null.TryFind type' with + | None -> unexpectedToken tok parser + | Some nud -> nud + + nud tok parser |> expr + + let expr parser = + parser |> exprPwr 0 + + let exprSkip type' parser = + let expr = parser |> expr + parser |> skipIf type' + expr + + let rec exprList parser = + match !parser.Input with + | [] -> [] + | _ -> (parser |> expr) :: (parser |> exprList) + + let stmt term parser = + let token = parser |> current + match parser.Stmt.TryFind (token |> parser.Type) with + | Some stmt -> parser |> skip; stmt token parser + | None -> parser |> exprSkip term + + let rec stmtList term parser = + match !parser.Input with + | [] -> [] + | _ -> (parser |> stmt term) :: (parser |> stmtList term) + + let match' pattern parser = + let rec match' acc pattern parser = + match pattern with + | [] -> acc |> List.rev + + | Sym(symbol)::pattern -> + parser |> skipIf symbol + parser |> match' acc pattern + + | Get(f)::pattern -> + let acc = (f parser) :: acc + parser |> match' acc pattern + + parser |> match' [] pattern + + (* + Convenience functions exposed for + easing parser definition and usage + *) + + let create<'a, 'b, 'c when 'c : comparison> type' position prettyPrint = { + Input = ref [] + Lines = None + + Type = type' + Position = position + PrettyPrint = prettyPrint + + BindingPower = Map.empty<'c, int> + Null = Map.empty<'c, 'a -> T<'a, 'b, 'c> -> 'b> + Stmt = Map.empty<'c, 'a -> T<'a, 'b, 'c> -> 'b> + Left = Map.empty<'c, 'a -> 'b -> T<'a, 'b, 'c> -> 'b> + } + + let matchError () = exn "Match pattern failed" + let smd token funct parser = {parser with T.Stmt = parser.Stmt.Add(token, funct)} + let nud token funct parser = {parser with T.Null = parser.Null.Add(token, funct)} + let led token funct parser = {parser with T.Left = parser.Left.Add(token, funct)} + let bpw token power parser = {parser with T.BindingPower = parser.BindingPower.Add(token, power)} + + (*Defines a left-associative infix operator*) + let infix f typ pwr p = + let infix tok left p = + f tok left (p |> exprPwr pwr) + + p |> bpw typ pwr |> led typ infix + + (*Defines a right-associative infix operator*) + let infixr f typ pwr p = + let lpwr = pwr - 1 + + let infix tok left p = + f tok left (p |> exprPwr lpwr) + + p |> bpw typ pwr |> led typ infix + + (*Defines a prefix/unary operator*) + let prefix f typ pwr p = + let prefix tok parser = + f tok (parser |> exprPwr pwr) + + p |> nud typ prefix + + (*Defines a constant*) + let constant symbol value p = + p |> nud symbol (fun _ _ -> value) + + (* + Runs the parser and treats all + top level construct as expressions + *) + let runExpr input source parser = + {parser with + T.Input = ref input + T.Lines = source + } |> exprList + + (* + Runs the parser and treats all + top level construct as statements + *) + let runStmt input source term parser = + {parser with + T.Input = ref input + T.Lines = source + } |> stmtList term + +(* + Example parser for a very simple grammar +*) + +//AST Types +type UnaryOp + = Plus + | Minus + +type BinaryOp + = Multiply + | Add + | Subtract + | Divide + | Assign + | Equals + +type Ast + = Number of int + | Identifier of string + | String of string + | Binary of BinaryOp * Ast * Ast + | Unary of UnaryOp * Ast + | Ternary of Ast * Ast * Ast // test * ifTrue * ifFalse + | If of Ast * Ast * Ast option // test + ifTrue and possibly ifFalse (else branch) + | Call of Ast * Ast list // target + arguments list + | Block of Ast list // statements list + | True + | False + +//Shorthand types for convenience +module P = Parser +type Token = string * string * (Parser.Pos) +type P = Parser.T + +//Utility functions for extracting values out of Token +let type' ((t, _, _):Token) = t +let value ((_, v, _):Token) = v +let pos ((_, _, p):Token) = p +let value_num (t:Token) = t |> value |> int + +//Utility functions for creating new tokens +let number value pos : Token = "#number", value, pos +let string value pos : Token = "#string", value, pos +let identifier name pos : Token = "#identifier", name, pos + +let symbol type' pos : Token = type', "", pos +let add = symbol "+" +let sub = symbol "-" +let mul = symbol "*" +let div = symbol "/" +let assign = symbol "=" +let equals = symbol "==" +let lparen = symbol "(" +let rparen = symbol ")" +let lbrace = symbol "{" +let rbrace = symbol "}" +let comma = symbol "," +let qmark = symbol "?" +let colon = symbol ":" +let scolon = symbol ";" +let if' = symbol "if" +let true' = symbol "true" +let else' = symbol "else" + +//Utility functions for converting tokens to binary and unary operators +let toBinaryOp tok = + match type' tok with + | "=" -> BinaryOp.Assign + | "+" -> BinaryOp.Add + | "-" -> BinaryOp.Subtract + | "*" -> BinaryOp.Multiply + | "/" -> BinaryOp.Divide + | "==" -> BinaryOp.Equals + | _ -> P.exn (sprintf "Couldn't convert %s-token to BinaryOp" (type' tok)) + +let toUnaryOp tok = + match type' tok with + | "+" -> UnaryOp.Plus + | "-" -> UnaryOp.Minus + | _ -> P.exn (sprintf "Couldn't convert %s-token to UnaryOp" (type' tok)) + +//Utility function for defining infix operators +let infix = + P.infix (fun token left right -> + Binary(token |> toBinaryOp, left, right)) + +//Utility function for defining prefix operators +let prefix = + P.prefix (fun token ast -> + Unary(token |> toUnaryOp, ast)) + +//Utility function for defining constants +let constant typ value p = + p |> P.nud typ (fun _ _ -> value) + +//Utility function for parsing a block +let block p = + let rec stmts p = + match p |> P.currentTypeTry with + | None -> [] + | Some "}" -> p |> P.skip; [] + | _ -> (p |> P.stmt ";") :: (stmts p) + + p |> P.skipIf "{" + Block(p |> stmts) + +//Pretty printing function for error messages +let prettyPrint (tok:Token) = + match tok with + | "#number", value, _ -> sprintf "#number %s" value + | "#identifier", name, _ -> sprintf "#identifier %s" name + | "#string", value, _ -> sprintf "#string \"%s\"" value + | type', _, _ -> type' + +//The parser definition +let example_parser = + (P.create type' pos (Some prettyPrint)) + + //Literals and identifiers + |> P.nud "#number" (fun t _ -> t |> value |> int |> Number) + |> P.nud "#identifier" (fun t _ -> t |> value |> Identifier) + |> P.nud "#string" (fun t _ -> t |> value |> String) + + //Constants + |> constant "true" Ast.True + |> constant "false" Ast.False + + //Infix Operators + |> infix "==" 40 + |> infix "+" 50 + |> infix "-" 50 + |> infix "*" 60 + |> infix "/" 60 + |> infix "=" 80 + + //Prefix Operators + |> prefix "+" 70 + |> prefix "-" 70 + + //Grouping expressions () + |> P.nud "(" (fun t p -> p |> P.exprSkip ")") + + //Ternary operator ? : + |> P.bpw "?" 70 + |> P.led "?" (fun _ left p -> + let ternary = [P.Get P.expr; P.Sym ":"; P.Get P.expr] + match p |> P.match' ternary with + | ifTrue::ifFalse::_ -> Ternary(left, ifTrue, ifFalse) + | _ -> P.matchError() + ) + + //If/Else statement if() { }] + |> P.smd "if" (fun _ p -> + let if' = [P.Sym "("; P.Get P.expr; P.Sym ")"; P.Get block] + let else' = [P.Sym "else"; P.Get block] + + match p |> P.match' if' with + | test::ifTrue::_ -> + match p |> P.match' else' with + | ifFalse::_ -> If(test, ifTrue, Some(ifFalse)) + | _ -> If(test, ifTrue, None) + | _ -> P.matchError() + ) + + //Function call + |> P.bpw "(" 80 + |> P.led "(" (fun _ func p -> + let rec args (p:P) = + match p |> P.currentType with + | ")" -> p |> P.skip; [] + | "," -> p |> P.skip; args p + | _ -> (p |> P.expr) :: args p + + Call(func, args p) + ) + +//Code to parse +(* +1: x = 5; +2: y = 5; +3: +4: if(x == y) { +5: print("x equals y"); +6: } else { +7: print("x doesn't equal y"); +8: } +*) + +let code = @"x = 5; +y = 5; + +if(x == y) { + print('x equals y'); +} else { + print('x doesn't equal y'); +}" + +//The code in tokens, manually entered +//since we don't have a lexer to produce +//the tokens for us +let tokens = + [ + //x = 5; + identifier "x" (1, 1) + assign (1, 3) + number "5" (1, 5) + scolon (1, 6) + + //y = 5; + identifier "y" (2, 1) + assign (2, 3) + number "5" (2, 5) + scolon (2, 6) + + //if(x == y) { + if' (4, 1) + lparen (4, 3) + identifier "x" (4, 4) + equals (4, 6) + identifier "y" (4, 9) + rparen (4, 10) + lbrace (4, 12) + + //print("x equals y"); + identifier "print" (5, 3) + lparen (5, 8) + string "x equals y" (5, 9) + rparen (5, 21) + scolon (5, 22) + + //} else { + rbrace (6, 1) + else' (6, 3) + lbrace (6, 7) + + //print("x doesn't equal y"); + identifier "print" (7, 3) + lparen (7, 7) + string "x doesn't equal y" (7, 9) + rparen (7, 27) + scolon (7, 28) + + //} + rbrace (8, 1) + ] + +let ast = example_parser |> P.runStmt tokens (Some code) ";" diff --git a/tests/service/Common.fs b/tests/service/Common.fs index 080049d5f92..15d1ea4d24b 100644 --- a/tests/service/Common.fs +++ b/tests/service/Common.fs @@ -353,8 +353,8 @@ let getSymbolUsesFromSource (source: string) = let _, typeCheckResults = getParseAndCheckResults source typeCheckResults.GetAllUsesOfAllSymbolsInFile() -let getSymbols (symbolUses: FSharpSymbolUse[]) = - symbolUses |> Array.map (fun symbolUse -> symbolUse.Symbol) +let getSymbols (symbolUses: seq) = + symbolUses |> Seq.map (fun symbolUse -> symbolUse.Symbol) let getSymbolName (symbol: FSharpSymbol) = @@ -371,25 +371,25 @@ let getSymbolName (symbol: FSharpSymbol) = let assertContainsSymbolWithName name source = getSymbols source - |> Array.choose getSymbolName - |> Array.contains name + |> Seq.choose getSymbolName + |> Seq.contains name |> shouldEqual true let assertContainsSymbolsWithNames (names: string list) source = let symbolNames = getSymbols source - |> Array.choose getSymbolName + |> Seq.choose getSymbolName for name in names do symbolNames - |> Array.contains name + |> Seq.contains name |> shouldEqual true let assertHasSymbolUsages (names: string list) (results: FSharpCheckFileResults) = let symbolNames = getSymbolUses results |> getSymbols - |> Array.choose getSymbolName + |> Seq.choose getSymbolName |> set for name in names do @@ -398,7 +398,7 @@ let assertHasSymbolUsages (names: string list) (results: FSharpCheckFileResults) let findSymbolUseByName (name: string) (results: FSharpCheckFileResults) = getSymbolUses results - |> Array.find (fun symbolUse -> + |> Seq.find (fun symbolUse -> match getSymbolName symbolUse.Symbol with | Some symbolName -> symbolName = name | _ -> false) diff --git a/tests/service/EditorTests.fs b/tests/service/EditorTests.fs index b10ee676460..0c43499b396 100644 --- a/tests/service/EditorTests.fs +++ b/tests/service/EditorTests.fs @@ -625,6 +625,7 @@ type DU = Case1 let file = "/home/user/Test.fsx" let parseResult, typeCheckResults = parseAndCheckScript(file, input) typeCheckResults.GetAllUsesOfAllSymbolsInFile() + |> Array.ofSeq |> Array.map (fun su -> let r = su.RangeAlternate r.StartLine, r.StartColumn, r.EndLine, r.EndColumn) @@ -643,6 +644,7 @@ let _ = arr.[..number2] let file = "/home/user/Test.fsx" let parseResult, typeCheckResults = parseAndCheckScript(file, input) typeCheckResults.GetAllUsesOfAllSymbolsInFile() + |> Array.ofSeq |> Array.map (fun su -> let r = su.RangeAlternate su.Symbol.ToString(), (r.StartLine, r.StartColumn, r.EndLine, r.EndColumn)) @@ -679,6 +681,7 @@ let test3 = System.Text.RegularExpressions.RegexOptions.Compiled let allSymbols = typeCheckResults.GetAllUsesOfAllSymbolsInFile() let enums = allSymbols + |> Array.ofSeq |> Array.choose(fun s -> match s.Symbol with :? FSharpEntity as e when e.IsEnum -> Some e | _ -> None) |> Array.distinct |> Array.map(fun e -> (e.DisplayName, e.FSharpFields @@ -729,6 +732,7 @@ let _ = let file = "/home/user/Test.fsx" let _, typeCheckResults = parseAndCheckScript(file, input) typeCheckResults.GetAllUsesOfAllSymbolsInFile() + |> Array.ofSeq |> Array.map (fun su -> let r = su.RangeAlternate su.Symbol.ToString(), (r.StartLine, r.StartColumn, r.EndLine, r.EndColumn)) @@ -775,6 +779,7 @@ type Class1() = let file = "/home/user/Test.fsx" let _, typeCheckResults = parseAndCheckScript(file, input) typeCheckResults.GetAllUsesOfAllSymbolsInFile() + |> Array.ofSeq |> Array.map (fun su -> let r = su.RangeAlternate su.Symbol.ToString(), (r.StartLine, r.StartColumn, r.EndLine, r.EndColumn)) @@ -824,6 +829,7 @@ let x: T() let file = "/home/user/Test.fsx" let _, typeCheckResults = parseAndCheckScript(file, input) typeCheckResults.GetAllUsesOfAllSymbolsInFile() + |> Array.ofSeq |> Array.map (fun su -> let r = su.RangeAlternate let isConstructor = @@ -1137,6 +1143,7 @@ let _ = Threading.Buzz = null let file = "/home/user/Test.fsx" let _, typeCheckResults = parseAndCheckScript(file, input) typeCheckResults.GetAllUsesOfAllSymbolsInFile() + |> Array.ofSeq |> Array.map (fun su -> let r = su.RangeAlternate su.Symbol.ToString(), (r.StartLine, r.StartColumn, r.EndLine, r.EndColumn)) @@ -1352,6 +1359,7 @@ let ``FSharpField.IsNameGenerated`` () = let symbols = typeCheckResults.GetAllUsesOfAllSymbolsInFile() symbols + |> Array.ofSeq |> Array.choose (fun su -> match su.Symbol with | :? FSharpEntity as entity -> Some entity.FSharpFields diff --git a/tests/service/ProjectAnalysisTests.fs b/tests/service/ProjectAnalysisTests.fs index 57486501ecc..09b360b66ee 100644 --- a/tests/service/ProjectAnalysisTests.fs +++ b/tests/service/ProjectAnalysisTests.fs @@ -3414,7 +3414,7 @@ let ``Test Project24 all symbols`` () = let allUses = backgroundTypedParse1.GetAllUsesOfAllSymbolsInFile() - + |> Array.ofSeq |> Array.map (fun s -> (s.Symbol.DisplayName, Project24.cleanFileName s.FileName, tups s.RangeAlternate, attribsOfSymbolUse s, attribsOfSymbol s.Symbol)) allUses |> shouldEqual @@ -3521,7 +3521,7 @@ let ``Test symbol uses of properties with both getters and setters`` () = let getAllSymbolUses = backgroundTypedParse1.GetAllUsesOfAllSymbolsInFile() - + |> Array.ofSeq |> Array.map (fun s -> (s.Symbol.DisplayName, Project24.cleanFileName s.FileName, tups s.RangeAlternate, attribsOfSymbol s.Symbol)) getAllSymbolUses |> shouldEqual @@ -3669,7 +3669,7 @@ let ``Test Project25 symbol uses of type-provided members`` () = let allUses = backgroundTypedParse1.GetAllUsesOfAllSymbolsInFile() - + |> Array.ofSeq |> Array.map (fun s -> (s.Symbol.FullName, Project25.cleanFileName s.FileName, tups s.RangeAlternate, attribsOfSymbol s.Symbol)) allUses |> shouldEqual diff --git a/tests/service/Symbols.fs b/tests/service/Symbols.fs index 4ecd5b372c0..d2ca83a99be 100644 --- a/tests/service/Symbols.fs +++ b/tests/service/Symbols.fs @@ -38,6 +38,7 @@ match "foo" with let _, checkResults = parseAndCheckFile fileName source options checkResults.GetAllUsesOfAllSymbolsInFile() + |> Array.ofSeq |> Array.filter (fun su -> su.RangeAlternate.StartLine = line && su.Symbol :? FSharpActivePatternCase) |> Array.map (fun su -> su.Symbol :?> FSharpActivePatternCase) @@ -118,10 +119,11 @@ let x = 123 let _, checkResults = parseAndCheckFile fileName source options checkResults.GetAllUsesOfAllSymbolsInFile() - |> Array.tryFind (fun su -> su.Symbol.DisplayName = "x") - |> Option.orElseWith (fun _ -> failwith "Could not get symbol") - |> Option.map (fun su -> su.Symbol :?> FSharpMemberOrFunctionOrValue) - |> Option.iter (fun symbol -> symbol.Attributes.Count |> shouldEqual 1) + |> Array.ofSeq + |> Array.tryFind (fun su -> su.Symbol.DisplayName = "x") + |> Option.orElseWith (fun _ -> failwith "Could not get symbol") + |> Option.map (fun su -> su.Symbol :?> FSharpMemberOrFunctionOrValue) + |> Option.iter (fun symbol -> symbol.Attributes.Count |> shouldEqual 1) module Types = [] diff --git a/vsintegration/src/FSharp.Editor/CodeFix/SimplifyName.fs b/vsintegration/src/FSharp.Editor/CodeFix/SimplifyName.fs index a2768640387..4656ce1c42b 100644 --- a/vsintegration/src/FSharp.Editor/CodeFix/SimplifyName.fs +++ b/vsintegration/src/FSharp.Editor/CodeFix/SimplifyName.fs @@ -6,12 +6,9 @@ open System.Composition open System.Collections.Immutable open System.Threading.Tasks -open Microsoft.CodeAnalysis -open Microsoft.CodeAnalysis.Diagnostics open Microsoft.CodeAnalysis.Text open Microsoft.CodeAnalysis.CodeFixes open Microsoft.CodeAnalysis.ExternalAccess.FSharp.Diagnostics -open SymbolHelpers [] type internal FSharpSimplifyNameCodeFixProvider() = diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index 86484f79739..a68ce44a226 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -5,6 +5,7 @@ module internal Microsoft.VisualStudio.FSharp.Editor.Extensions open System open System.IO +open System.Collections.Immutable open Microsoft.CodeAnalysis open Microsoft.CodeAnalysis.Text @@ -243,7 +244,6 @@ module Option = [] module Seq = - open System.Collections.Immutable let toImmutableArray (xs: seq<'a>) : ImmutableArray<'a> = xs.ToImmutableArray() @@ -257,6 +257,8 @@ module Array = i <- i + 1 state + let toImmutableArray (xs: 'T[]) = xs.ToImmutableArray() + [] module Exception = diff --git a/vsintegration/src/FSharp.Editor/Diagnostics/SimplifyNameDiagnosticAnalyzer.fs b/vsintegration/src/FSharp.Editor/Diagnostics/SimplifyNameDiagnosticAnalyzer.fs index e888ccdc7ce..43641cc957b 100644 --- a/vsintegration/src/FSharp.Editor/Diagnostics/SimplifyNameDiagnosticAnalyzer.fs +++ b/vsintegration/src/FSharp.Editor/Diagnostics/SimplifyNameDiagnosticAnalyzer.fs @@ -7,18 +7,13 @@ open System.Composition open System.Collections.Immutable open System.Diagnostics open System.Threading -open System.Threading.Tasks open Microsoft.CodeAnalysis -open Microsoft.CodeAnalysis.Diagnostics -open FSharp.Compiler open FSharp.Compiler.Range open System.Runtime.Caching -open Microsoft.CodeAnalysis.Host.Mef open Microsoft.CodeAnalysis.ExternalAccess.FSharp.Diagnostics open FSharp.Compiler.SourceCodeServices -type private TextVersionHash = int type private PerDocumentSavedData = { Hash: int; Diagnostics: ImmutableArray } [)>] @@ -36,7 +31,7 @@ type internal SimplifyNameDiagnosticAnalyzer [] () = interface IFSharpSimplifyNameDiagnosticAnalyzer with - member this.AnalyzeSemanticsAsync(descriptor, document: Document, cancellationToken: CancellationToken) = + member _.AnalyzeSemanticsAsync(descriptor, document: Document, cancellationToken: CancellationToken) = asyncMaybe { do! Option.guard document.FSharpOptions.CodeFixes.SimplifyName do Trace.TraceInformation("{0:n3} (start) SimplifyName", DateTime.Now.TimeOfDay.TotalSeconds) diff --git a/vsintegration/src/FSharp.Editor/Diagnostics/UnusedDeclarationsAnalyzer.fs b/vsintegration/src/FSharp.Editor/Diagnostics/UnusedDeclarationsAnalyzer.fs index 66c0a60e452..62b556cb0bb 100644 --- a/vsintegration/src/FSharp.Editor/Diagnostics/UnusedDeclarationsAnalyzer.fs +++ b/vsintegration/src/FSharp.Editor/Diagnostics/UnusedDeclarationsAnalyzer.fs @@ -25,7 +25,7 @@ type internal UnusedDeclarationsAnalyzer [] () = interface IFSharpUnusedDeclarationsDiagnosticAnalyzer with - member __.AnalyzeSemanticsAsync(descriptor, document, cancellationToken) = + member _.AnalyzeSemanticsAsync(descriptor, document, cancellationToken) = asyncMaybe { do! Option.guard document.FSharpOptions.CodeFixes.UnusedDeclarations diff --git a/vsintegration/src/FSharp.Editor/Diagnostics/UnusedOpensDiagnosticAnalyzer.fs b/vsintegration/src/FSharp.Editor/Diagnostics/UnusedOpensDiagnosticAnalyzer.fs index f9e4d917c2d..a16a66e43eb 100644 --- a/vsintegration/src/FSharp.Editor/Diagnostics/UnusedOpensDiagnosticAnalyzer.fs +++ b/vsintegration/src/FSharp.Editor/Diagnostics/UnusedOpensDiagnosticAnalyzer.fs @@ -7,19 +7,12 @@ open System.Composition open System.Collections.Immutable open System.Diagnostics open System.Threading -open System.Threading.Tasks open Microsoft.CodeAnalysis -open Microsoft.CodeAnalysis.Text -open Microsoft.CodeAnalysis.Diagnostics -open FSharp.Compiler open FSharp.Compiler.Range open FSharp.Compiler.SourceCodeServices -open FSharp.Compiler.SyntaxTree -open Microsoft.VisualStudio.FSharp.Editor.Symbols -open Microsoft.CodeAnalysis.Host.Mef open Microsoft.CodeAnalysis.ExternalAccess.FSharp.Diagnostics [)>]