Add a couple of "stress tests" to the integration test project (#11481)

Two tests that run for an hour, adding and removing components and
verifying that classification is correct afterwards. One uses an RCL,
one doesn't.

The tests are setup to not run in CI, only manually on dev machines, so
as not to delay the CI runs, and because sometimes these have a
different version of "success" :)

Author: davidwengier

azure-pipelines-conditional-integration.yml

@@ -18,6 +18,8 @@ variables:
   value: false
 - name: Codeql.SkipTaskAutoInjection
   value: true
+- name: _IntegrationTestsRunningInCI
+  value: true
 
 trigger: none
 

azure-pipelines-integration-dartlab.yml

@@ -35,6 +35,8 @@ variables:
   value: false
 - name: Codeql.SkipTaskAutoInjection
   value: true
+- name: _IntegrationTestsRunningInCI
+  value: true
 
 stages:
 - template: \stages\visual-studio\agent.yml@DartLabTemplates

azure-pipelines.yml

@@ -20,6 +20,8 @@ variables:
   value: false
 - name: Codeql.SkipTaskAutoInjection
   value: true
+- name: _IntegrationTestsRunningInCI
+  value: true
 
 trigger:
   batch: true

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractIntegrationTest.cs

@@ -31,6 +31,8 @@ public abstract class AbstractIntegrationTest : AbstractIdeIntegrationTest
 {
     protected CancellationToken ControlledHangMitigatingCancellationToken => HangMitigatingCancellationToken;
 
+    protected virtual bool AllowDebugFails => false;
+
     public override async Task InitializeAsync()
     {
         // Not sure why the module initializer doesn't seem to work for integration tests
@@ -41,11 +43,14 @@ public override async Task InitializeAsync()
 
     public override void Dispose()
     {
-        var fails = ThrowingTraceListener.Fails;
-        Assert.False(fails.Length > 0, $"""
-            Expected 0 Debug.Fail calls. Actual:
-            {string.Join(Environment.NewLine, fails)}
-            """);
+        if (!AllowDebugFails)
+        {
+            var fails = ThrowingTraceListener.Fails;
+            Assert.False(fails.Length > 0, $"""
+                Expected 0 Debug.Fail calls. Actual:
+                {string.Join(Environment.NewLine, fails)}
+                """);
+        }
 
         base.Dispose();
     }

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/AbstractRazorEditorTest.cs

@@ -32,6 +32,10 @@ public abstract class AbstractRazorEditorTest(ITestOutputHelper testOutput) : Ab
 
     protected virtual string TargetFrameworkElement => $"""<TargetFramework>{TargetFramework}</TargetFramework>""";
 
+    protected virtual string ProjectZipFile => "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProject.zip";
+
+    private protected virtual ILogger Logger => _testLogger.AssumeNotNull();
+
     protected string ProjectFilePath => _projectFilePath.AssumeNotNull();
 
     public override async Task InitializeAsync()
@@ -96,19 +100,20 @@ private async Task<string> CreateAndOpenBlazorProjectAsync(CancellationToken can
 
         var solutionPath = CreateTemporaryPath();
 
-        var resourceName = "Microsoft.VisualStudio.Razor.IntegrationTests.TestFiles.BlazorProject.zip";
-        using var zipStream = typeof(AbstractRazorEditorTest).Assembly.GetManifestResourceStream(resourceName);
+        using var zipStream = typeof(AbstractRazorEditorTest).Assembly.GetManifestResourceStream(ProjectZipFile);
         using var zip = new ZipArchive(zipStream);
         zip.ExtractToDirectory(solutionPath);
 
         var slnFile = Directory.EnumerateFiles(solutionPath, "*.sln").Single();
-        var projectFile = Directory.EnumerateFiles(solutionPath, "*.csproj", SearchOption.AllDirectories).Single();
 
-        PrepareProjectForFirstOpen(projectFile);
+        foreach (var projectFile in Directory.EnumerateFiles(solutionPath, "*.csproj", SearchOption.AllDirectories))
+        {
+            PrepareProjectForFirstOpen(projectFile);
+        }
 
         await TestServices.SolutionExplorer.OpenSolutionAsync(slnFile, cancellationToken);
 
-        return projectFile;
+        return Directory.EnumerateFiles(solutionPath, $"{RazorProjectConstants.BlazorProjectName}.csproj", SearchOption.AllDirectories).Single();
     }
 
     protected virtual void PrepareProjectForFirstOpen(string projectFileName)

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Classification.cs

@@ -20,8 +20,10 @@ internal partial class EditorInProcess
     /// </summary>
     /// <param name="cancellationToken">A cancellation token.</param>
     /// <param name="count">The number of the given classification to expect.</param>
+    /// <param name="exact">Whether to wait for exactly <paramref name="count"/> classifications.</param>
     /// <returns>A <see cref="Task"/> which completes when classification is "ready".</returns>
-    public Task WaitForComponentClassificationAsync(CancellationToken cancellationToken, int count = 1) => WaitForSemanticClassificationAsync("RazorComponentElement", cancellationToken, count);
+    public Task WaitForComponentClassificationAsync(CancellationToken cancellationToken, int count = 1, bool exact = false)
+        => WaitForSemanticClassificationAsync("RazorComponentElement", cancellationToken, count, exact);
 
     /// <summary>
     /// Waits for any semantic classifications to be available on the active TextView, and for at least one of the
@@ -30,8 +32,9 @@ internal partial class EditorInProcess
     /// <param name="cancellationToken">A cancellation token.</param>
     /// <param name="expectedClassification">The classification to wait for, if any.</param>
     /// <param name="count">The number of the given classification to expect.</param>
+    /// <param name="exact">Whether to wait for exactly <paramref name="count"/> classifications.</param>
     /// <returns>A <see cref="Task"/> which completes when classification is "ready".</returns>
-    public async Task WaitForSemanticClassificationAsync(string expectedClassification, CancellationToken cancellationToken, int count = 1)
+    public async Task WaitForSemanticClassificationAsync(string expectedClassification, CancellationToken cancellationToken, int count = 1, bool exact = false)
     {
         var textView = await TestServices.Editor.GetActiveTextViewAsync(cancellationToken);
         var classifier = await GetClassifierAsync(textView, cancellationToken);
@@ -42,7 +45,7 @@ public async Task WaitForSemanticClassificationAsync(string expectedClassificati
         classifier.ClassificationChanged += Classifier_ClassificationChanged;
 
         // Check that we're not ALREADY changed
-        if (HasClassification(classifier, textView, expectedClassification, count))
+        if (HasClassification(classifier, textView, expectedClassification, count, exact))
         {
             semaphore.Release();
             classifier.ClassificationChanged -= Classifier_ClassificationChanged;
@@ -60,13 +63,13 @@ public async Task WaitForSemanticClassificationAsync(string expectedClassificati
 
         void Classifier_ClassificationChanged(object sender, ClassificationChangedEventArgs e)
         {
-            if (HasClassification(classifier, textView, expectedClassification, count))
+            if (HasClassification(classifier, textView, expectedClassification, count, exact))
             {
                 semaphore.Release();
             }
         }
 
-        static bool HasClassification(IClassifier classifier, ITextView textView, string expectedClassification, int count)
+        static bool HasClassification(IClassifier classifier, ITextView textView, string expectedClassification, int count, bool exact)
         {
             var classifications = GetClassifications(classifier, textView);
 
@@ -80,7 +83,8 @@ static bool HasClassification(IClassifier classifier, ITextView textView, string
                 }
             }
 
-            return found >= count;
+            return found == count ||
+                (!exact && found > count);
         }
 
         static bool ClassificationMatches(string expectedClassification, IClassificationType classificationType)

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_Commands.cs

@@ -58,7 +58,8 @@ public async Task InvokeRenameAsync(CancellationToken cancellationToken)
         var commandId = VSStd2KCmdID.RENAME;
 
         // Rename seems to be extra-succeptable to COM exceptions
-        await Helper.RetryAsync<bool?>(async (cancellationToken) => {
+        await Helper.RetryAsync<bool?>(async (cancellationToken) =>
+        {
             await ExecuteCommandAsync(commandGuid, (uint)commandId, cancellationToken);
             return true;
         }, TimeSpan.FromSeconds(1), cancellationToken);
@@ -69,15 +70,18 @@ public async Task CloseCodeFileAsync(string projectName, string relativeFilePath
         await CloseFileAsync(projectName, relativeFilePath, VSConstants.LOGVIEWID.Code_guid, saveFile, cancellationToken);
     }
 
-    public async Task CloseCurrentlyFocusedWindowAsync(CancellationToken cancellationToken)
+    public async Task CloseCurrentlyFocusedWindowAsync(CancellationToken cancellationToken, bool save = false)
     {
         await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken);
 
         var monitorSelection = await GetRequiredGlobalServiceAsync<SVsShellMonitorSelection, IVsMonitorSelection>(cancellationToken);
         ErrorHandler.ThrowOnFailure(monitorSelection.GetCurrentElementValue((uint)VSSELELEMID.SEID_WindowFrame, out var windowFrameObj));
         var windowFrame = (IVsWindowFrame)windowFrameObj;
 
-        ErrorHandler.ThrowOnFailure(windowFrame.CloseFrame((uint)__FRAMECLOSE.FRAMECLOSE_NoSave));
+        var closeFlags = save
+            ? __FRAMECLOSE.FRAMECLOSE_SaveIfDirty
+            : __FRAMECLOSE.FRAMECLOSE_NoSave;
+        ErrorHandler.ThrowOnFailure(windowFrame.CloseFrame((uint)closeFlags));
     }
 
     private async Task ExecuteCommandAsync(Guid commandGuid, uint commandId, CancellationToken cancellationToken)

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/InProcess/EditorInProcess_LightBulb.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.VisualStudio.Language.Intellisense;
@@ -24,6 +25,15 @@ public async Task DismissLightBulbSessionAsync(CancellationToken cancellationTok
         broker.DismissSession(view);
     }
 
+    public async Task InvokeCodeActionAsync(string codeActionTitle, CancellationToken cancellationToken)
+    {
+        var codeActions = await ShowLightBulbAsync(cancellationToken);
+
+        var codeAction = codeActions.First(a => a.Actions.Single().DisplayText == codeActionTitle).Actions.Single();
+
+        await InvokeCodeActionAsync(codeAction, cancellationToken);
+    }
+
     public async Task<IEnumerable<SuggestedActionSet>> InvokeCodeActionListAsync(CancellationToken cancellationToken)
     {
         var lightbulbs = await ShowLightBulbAsync(cancellationToken);

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/ManualRunOnlyIdeFactAttribute.cs

@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using Xunit;
+
+namespace Microsoft.VisualStudio.Razor.IntegrationTests;
+
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public class ManualRunOnlyIdeFactAttribute : IdeFactAttribute
+{
+    public ManualRunOnlyIdeFactAttribute()
+    {
+        if (Environment.GetEnvironmentVariable("_IntegrationTestsRunningInCI") is not null)
+        {
+            Skip = "This test can only run manually";
+        }
+    }
+}

src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/StressTests/AbstractStressTest.cs

@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT license. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Razor.Logging;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.VisualStudio.Razor.IntegrationTests;
+
+[IdeSettings(MinVersion = VisualStudioVersion.VS2022, RootSuffix = "RoslynDev", MaxAttempts = 1)]
+public abstract class AbstractStressTest(ITestOutputHelper testOutputHelper) : AbstractRazorEditorTest(testOutputHelper)
+{
+    protected override bool AllowDebugFails => true;
+
+    protected Task RunStressTestAsync(Func<int, CancellationToken, Task> iterationFunc)
+        => RunStressTestAsync(iterationFunc, TimeSpan.FromHours(1), TimeSpan.FromMinutes(1));
+
+    protected async Task RunStressTestAsync(Func<int, CancellationToken, Task> iterationFunc, TimeSpan maxRunTime, TimeSpan iterationTimeout)
+    {
+        var min = long.MaxValue;
+        var max = long.MinValue;
+        var avg = 0L;
+
+        var i = 0;
+        var start = DateTime.Now;
+        while (DateTime.Now.Subtract(maxRunTime) < start)
+        {
+            var iterationStart = Stopwatch.GetTimestamp();
+            Logger.LogInformation($"**** Test iteration started: #{i} at {DateTime.Now}");
+
+            using var iterationCts = new CancellationTokenSource(iterationTimeout);
+            var iterationToken = iterationCts.Token;
+
+            await iterationFunc(i, iterationToken);
+            i++;
+
+            var duration = Stopwatch.GetTimestamp() - iterationStart;
+            min = Math.Min(min, duration);
+            max = Math.Max(max, duration);
+            avg = ((avg * (i - 1)) + duration) / i;
+            Logger.LogInformation($"**** Test iteration finished: #{i} in {TimeSpan.FromTicks(duration).TotalMilliseconds}ms");
+            Logger.LogInformation($"**** Test iteration duration: min={TimeSpan.FromTicks(min).TotalMilliseconds}ms, max={TimeSpan.FromTicks(max).TotalMilliseconds}ms, avg={TimeSpan.FromTicks(avg).TotalMilliseconds}ms");
+        }
+    }
+}