From c5473368539d494e35ed5c06d0597d37ecdde3c9 Mon Sep 17 00:00:00 2001 From: Remy Willems Date: Thu, 24 Aug 2023 16:06:36 +0200 Subject: [PATCH] Improve performance for large projects (#4419) ### Changes - Sample sending notifications to the IDE, so at most one is sent every 100ms. Relatedly, `IdeState` is computed lazily, so it not computed for notifications that are not sent to the IDE. - Cancel ongoing compilation work and notification publishing immediately when UpdateDocument is entered - Start a new compilation slightly earlier, before computing the most recently changed ranges - Fix migration of recently changed ranges, so it properly accounts for Uri differences. - Make sure only one (correct) version of `WriterFromOutputHelper` is used in all test projects, to help in debugging non-deterministic failures - Small changes to `IncrementalVerificationDiagnosticsBetweenMethods` to help in debugging non-deterministic failures ### Testing - Add a test `QuickEditsInLargeFile` with a 1000 line and method file, that instantly makes a 100 small edits to that file, and asserts that the language server can process these changes in at most three times as much time as it takes to open the file. By submitting this pull request, I confirm that my contribution is made under the terms of the [MIT license](https://github.com/dafny-lang/dafny/blob/master/LICENSE.txt). --- Source/DafnyCore.Test/NodeTests.cs | 2 +- .../WriterFromOutputHelper.cs | 15 +-- Source/DafnyCore/DafnyFile.cs | 1 - Source/DafnyCore/Options/DafnyProject.cs | 60 +++++++--- Source/DafnyDriver/DafnyDoc.cs | 2 - .../DafnyLanguageServer.Test.csproj | 3 +- .../DafnyLanguageServerTestBase.cs | 2 + .../Performance/LargeFilesTest.cs | 111 ++++++++++++++++++ .../ProjectFiles/ProjectFilesTest.cs | 21 ++-- .../StandaloneServerTest.cs | 27 +---- .../Synchronization/DiagnosticsTest.cs | 5 +- .../Unit/GhostStateDiagnosticCollectorTest.cs | 3 +- .../Unit/TextDocumentLoaderTest.cs | 5 +- .../Util/ClientBasedLanguageServerTest.cs | 6 +- .../Util/Stringify.cs | 1 + .../Util/TestNotificationReceiver.cs | 2 +- .../Various/ExceptionTests.cs | 6 +- .../DafnyLanguageServer.cs | 3 - .../Language/CachingParser.cs | 1 - .../Language/IVerificationProgressReporter.cs | 2 +- .../Language/LanguageServerExtensions.cs | 1 - Source/DafnyLanguageServer/ServerCommand.cs | 10 +- .../Workspace/CompilationManager.cs | 21 +++- .../Workspace/Documents/Compilation.cs | 2 +- .../Documents/CompilationAfterParsing.cs | 4 +- .../Documents/CompilationAfterResolution.cs | 5 +- .../Workspace/ITextDocumentLoader.cs | 4 +- .../DafnyLanguageServer/Workspace/IdeState.cs | 42 ++++++- .../Workspace/IdeStateObserver.cs | 73 +++--------- .../Workspace/ProjectManager.cs | 58 +++++---- .../Workspace/TextDocumentLoader.cs | 13 +- .../Workspace/VerificationProgressReporter.cs | 18 ++- .../DafnyPipeline.Test.csproj | 3 +- Source/DafnyPipeline.Test/DocstringTest.cs | 4 +- .../DafnyPipeline.Test/FormatterBaseTest.cs | 1 + .../IntraMethodVerificationStability.cs | 9 +- Source/DafnyPipeline.Test/Issue1355.cs | 1 + Source/DafnyPipeline.Test/PluginsTest.cs | 1 + Source/DafnyPipeline.Test/RelativePaths.cs | 1 + Source/DafnyPipeline.Test/Trivia.cs | 1 + .../WriterFromOutputHelper.cs | 30 ----- Source/DafnyTestGeneration.Test/BasicTypes.cs | 1 + .../DafnyTestGeneration.Test/Collections.cs | 1 + .../DafnyTestGeneration.Test.csproj | 3 +- .../ShortCircuitRemoval.cs | 1 + Source/DafnyTestGeneration.Test/Various.cs | 1 + 46 files changed, 359 insertions(+), 228 deletions(-) rename Source/{DafnyTestGeneration.Test => DafnyCore.Test}/WriterFromOutputHelper.cs (66%) create mode 100644 Source/DafnyLanguageServer.Test/Performance/LargeFilesTest.cs delete mode 100644 Source/DafnyPipeline.Test/WriterFromOutputHelper.cs diff --git a/Source/DafnyCore.Test/NodeTests.cs b/Source/DafnyCore.Test/NodeTests.cs index e4a52d860f1..3e376b0b253 100644 --- a/Source/DafnyCore.Test/NodeTests.cs +++ b/Source/DafnyCore.Test/NodeTests.cs @@ -1,7 +1,7 @@ using System.Collections.Concurrent; using Microsoft.Dafny; -namespace DafnyCore.Test; +namespace DafnyCore.Test; public class NodeTests { diff --git a/Source/DafnyTestGeneration.Test/WriterFromOutputHelper.cs b/Source/DafnyCore.Test/WriterFromOutputHelper.cs similarity index 66% rename from Source/DafnyTestGeneration.Test/WriterFromOutputHelper.cs rename to Source/DafnyCore.Test/WriterFromOutputHelper.cs index 8448d66eb33..dcc5aed0027 100644 --- a/Source/DafnyTestGeneration.Test/WriterFromOutputHelper.cs +++ b/Source/DafnyCore.Test/WriterFromOutputHelper.cs @@ -1,13 +1,8 @@ -// Copyright by the contributors to the Dafny Project -// SPDX-License-Identifier: MIT - -#nullable disable -using System.IO; +#nullable enable using System.Text; -using Microsoft.Extensions.Primitives; using Xunit.Abstractions; -namespace DafnyTestGeneration.Test; +namespace DafnyCore.Test; public class WriterFromOutputHelper : TextWriter { private readonly StringBuilder buffer = new(); @@ -17,7 +12,7 @@ public WriterFromOutputHelper(ITestOutputHelper output) { this.output = output; } - public override void Write(string value) { + public override void Write(string? value) { if (value != null) { buffer.Append(value); } @@ -29,12 +24,12 @@ public override void Write(char value) { public override Encoding Encoding => Encoding.Default; - public override void WriteLine(string value) { + public override void WriteLine(string? value) { output.WriteLine(buffer + value); buffer.Clear(); } - public override void WriteLine(string format, params object[] arg) { + public override void WriteLine(string format, params object?[] arg) { output.WriteLine(buffer + format, arg); buffer.Clear(); } diff --git a/Source/DafnyCore/DafnyFile.cs b/Source/DafnyCore/DafnyFile.cs index 47800c4ae4d..af67b09e0f5 100644 --- a/Source/DafnyCore/DafnyFile.cs +++ b/Source/DafnyCore/DafnyFile.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Reflection.Metadata; using System.Reflection.PortableExecutable; using DafnyCore; diff --git a/Source/DafnyCore/Options/DafnyProject.cs b/Source/DafnyCore/Options/DafnyProject.cs index f1ad8784bc3..8576c45e229 100644 --- a/Source/DafnyCore/Options/DafnyProject.cs +++ b/Source/DafnyCore/Options/DafnyProject.cs @@ -56,37 +56,67 @@ public IEnumerable GetRootSourceUris(IFileSystem fileSystem) { if (!Uri.IsFile) { return new[] { Uri }; } + var matcher = GetMatcher(out var searchRoot); - var matcher = GetMatcher(); - - var diskRoot = Path.GetPathRoot(Uri.LocalPath); - var result = matcher.Execute(fileSystem.GetDirectoryInfoBase(diskRoot)); - var files = result.Files.Select(f => Path.Combine(diskRoot, f.Path)); + var result = matcher.Execute(fileSystem.GetDirectoryInfoBase(searchRoot)); + var files = result.Files.Select(f => Path.Combine(searchRoot, f.Path)); return files.Select(file => new Uri(Path.GetFullPath(file))); } public bool ContainsSourceFile(Uri uri) { - var fileSystemWithSourceFile = new InMemoryDirectoryInfoFromDotNet8(Path.GetPathRoot(uri.LocalPath)!, new[] { uri.LocalPath }); - return GetMatcher().Execute(fileSystemWithSourceFile).HasMatches; + var matcher = GetMatcher(out var searchRoot); + var fileSystemWithSourceFile = new InMemoryDirectoryInfoFromDotNet8(searchRoot, new[] { uri.LocalPath }); + return matcher.Execute(fileSystemWithSourceFile).HasMatches; } - private Matcher GetMatcher() { + private Matcher GetMatcher(out string commonRoot) { var projectRoot = Path.GetDirectoryName(Uri.LocalPath)!; - var root = Path.GetPathRoot(Uri.LocalPath)!; + var diskRoot = Path.GetPathRoot(Uri.LocalPath)!; + + var includes = Includes ?? new[] { "**/*.dfy" }; + var excludes = Excludes ?? Array.Empty(); + var fullPaths = includes.Concat(excludes).Select(p => Path.GetFullPath(p, projectRoot)).ToList(); + commonRoot = GetCommonParentDirectory(fullPaths) ?? diskRoot; var matcher = new Matcher(); - foreach (var includeGlob in Includes ?? new[] { "**/*.dfy" }) { - var fullPath = Path.GetFullPath(includeGlob, projectRoot); - matcher.AddInclude(Path.GetRelativePath(root, fullPath)); + foreach (var includeGlob in includes) { + matcher.AddInclude(Path.GetRelativePath(commonRoot, Path.GetFullPath(includeGlob, projectRoot))); } - foreach (var includeGlob in Excludes ?? Enumerable.Empty()) { - var fullPath = Path.GetFullPath(includeGlob, projectRoot); - matcher.AddExclude(Path.GetRelativePath(root, fullPath)); + foreach (var excludeGlob in excludes) { + matcher.AddExclude(Path.GetRelativePath(commonRoot, Path.GetFullPath(excludeGlob, projectRoot))); } return matcher; } + string GetCommonParentDirectory(IReadOnlyList strings) { + if (!strings.Any()) { + return null; + } + var commonPrefix = strings.FirstOrDefault() ?? ""; + + foreach (var newString in strings) { + var potentialMatchLength = Math.Min(newString.Length, commonPrefix.Length); + + if (potentialMatchLength < commonPrefix.Length) { + commonPrefix = commonPrefix.Substring(0, potentialMatchLength); + } + + for (var i = 0; i < potentialMatchLength; i++) { + if (newString[i] == '*' || newString[i] != commonPrefix[i]) { + commonPrefix = commonPrefix.Substring(0, i); + break; + } + } + } + + if (!Path.EndsInDirectorySeparator(commonPrefix)) { + commonPrefix = Path.GetDirectoryName(commonPrefix); + } + + return commonPrefix; + } + public void Validate(TextWriter outputWriter, IEnumerable