Add code for analzying copilot changes. (#78353)

Author: CyrusNajmabadi

src/EditorFeatures/Core.Wpf/Copilot/CopilotWpfTextCreationListener.cs

@@ -14,6 +14,7 @@
 using Microsoft.CodeAnalysis.Editor.Shared.Extensions;
 using Microsoft.CodeAnalysis.Editor.Shared.Utilities;
 using Microsoft.CodeAnalysis.Host.Mef;
+using Microsoft.CodeAnalysis.Options;
 using Microsoft.CodeAnalysis.Shared.TestHooks;
 using Microsoft.CodeAnalysis.Text;
 using Microsoft.CodeAnalysis.Threading;
@@ -29,27 +30,31 @@ namespace Microsoft.CodeAnalysis.Copilot;
 [TextViewRole(PredefinedTextViewRoles.Document)]
 internal sealed class CopilotWpfTextViewCreationListener : IWpfTextViewCreationListener
 {
+    private readonly IGlobalOptionService _globalOptions;
     private readonly IThreadingContext _threadingContext;
     private readonly Lazy<SuggestionServiceBase> _suggestionServiceBase;
     private readonly IAsynchronousOperationListener _listener;
 
-    private readonly AsyncBatchingWorkQueue<SuggestionAcceptedEventArgs> _workQueue;
+    private readonly AsyncBatchingWorkQueue<(bool accepted, ProposalBase proposal)> _completionWorkQueue;
 
     private int _started;
 
     [ImportingConstructor]
     [Obsolete(MefConstruction.ImportingConstructorMessage, error: true)]
     public CopilotWpfTextViewCreationListener(
+        IGlobalOptionService globalOptions,
         IThreadingContext threadingContext,
         Lazy<SuggestionServiceBase> suggestionServiceBase,
         IAsynchronousOperationListenerProvider listenerProvider)
     {
+        _globalOptions = globalOptions;
         _threadingContext = threadingContext;
         _suggestionServiceBase = suggestionServiceBase;
         _listener = listenerProvider.GetListener(FeatureAttribute.CopilotChangeAnalysis);
-        _workQueue = new AsyncBatchingWorkQueue<SuggestionAcceptedEventArgs>(
+
+        _completionWorkQueue = new AsyncBatchingWorkQueue<(bool accepted, ProposalBase proposal)>(
             DelayTimeSpan.Idle,
-            ProcessEventsAsync,
+            ProcessCompletionEventsAsync,
             _listener,
             _threadingContext.DisposalToken);
     }
@@ -64,30 +69,41 @@ public void TextViewCreated(IWpfTextView textView)
             Task.Run(() =>
             {
                 var suggestionService = _suggestionServiceBase.Value;
-                suggestionService.SuggestionAccepted += OnSuggestionAccepted;
+                suggestionService.SuggestionAccepted += OnCompletionSuggestionAccepted;
+                suggestionService.SuggestionDismissed += OnCompletionSuggestionDismissed;
             }).CompletesAsyncOperation(token);
         }
     }
 
-    private void OnSuggestionAccepted(object sender, SuggestionAcceptedEventArgs e)
+    private void OnCompletionSuggestionAccepted(object sender, SuggestionAcceptedEventArgs e)
+        => OnCompletionSuggestionEvent(accepted: true, e.FinalProposal);
+
+    private void OnCompletionSuggestionDismissed(object sender, SuggestionDismissedEventArgs e)
+        => OnCompletionSuggestionEvent(accepted: false, e.FinalProposal);
+
+    private void OnCompletionSuggestionEvent(bool accepted, ProposalBase? proposal)
     {
-        if (e.FinalProposal.Edits.Count == 0)
+        if (proposal is not { Edits.Count: > 0 })
             return;
 
-        _workQueue.AddWork(e);
+        _completionWorkQueue.AddWork((accepted, proposal));
     }
 
-    private async ValueTask ProcessEventsAsync(
-        ImmutableSegmentedList<SuggestionAcceptedEventArgs> list, CancellationToken cancellationToken)
+    private async ValueTask ProcessCompletionEventsAsync(
+        ImmutableSegmentedList<(bool accepted, ProposalBase proposal)> list, CancellationToken cancellationToken)
     {
-        foreach (var eventArgs in list)
-            await ProcessEventAsync(eventArgs, cancellationToken).ConfigureAwait(false);
+        // Ignore if analyzing changes is disabled for this user.
+        if (!_globalOptions.GetOption(CopilotOptions.AnalyzeCopilotChanges))
+            return;
+
+        foreach (var (accepted, proposal) in list)
+            await ProcessCompletionEventAsync(accepted, proposal, cancellationToken).ConfigureAwait(false);
     }
 
-    private static async ValueTask ProcessEventAsync(
-        SuggestionAcceptedEventArgs eventArgs, CancellationToken cancellationToken)
+    private static async ValueTask ProcessCompletionEventAsync(
+        bool accepted, ProposalBase proposal, CancellationToken cancellationToken)
     {
-        var proposal = eventArgs.FinalProposal;
+        const string featureId = "Completion";
         var proposalId = proposal.ProposalId;
 
         foreach (var editGroup in proposal.Edits.GroupBy(e => e.Span.Snapshot))
@@ -100,13 +116,22 @@ private static async ValueTask ProcessEventAsync(
             if (document is null)
                 continue;
 
+            // Currently we do not support analyzing languges other than C# and VB.  This is because we only want to do
+            // this analsis in our OOP process to avoid perf impact on the VS process.  And we don't have OOP for other
+            // languages yet.
+            if (!document.SupportsSemanticModel)
+                continue;
+
             var normalizedEdits = Normalize(editGroup);
             if (normalizedEdits.IsDefaultOrEmpty)
                 continue;
 
             var changeAnalysisService = document.Project.Solution.Services.GetRequiredService<ICopilotChangeAnalysisService>();
-            await changeAnalysisService.AnalyzeChangeAsync(
-                document, normalizedEdits, proposalId, cancellationToken).ConfigureAwait(false);
+            var analysisResult = await changeAnalysisService.AnalyzeChangeAsync(
+                document, normalizedEdits, cancellationToken).ConfigureAwait(false);
+
+            CopilotChangeAnalysisUtilities.LogCopilotChangeAnalysis(
+                featureId, accepted, proposalId, analysisResult, cancellationToken).Dispose();
         }
     }
 

src/EditorFeatures/TestUtilities/Diagnostics/DiagnosticTaggerWrapper.cs

@@ -9,7 +9,6 @@
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Editor.InlineDiagnostics;
-using Microsoft.CodeAnalysis.Editor.Shared.Utilities;
 using Microsoft.CodeAnalysis.Shared.TestHooks;
 using Microsoft.CodeAnalysis.SolutionCrawler;
 using Microsoft.CodeAnalysis.Test.Utilities;
@@ -23,7 +22,6 @@ internal sealed class DiagnosticTaggerWrapper<TProvider, TTag>
     where TTag : ITag
 {
     private readonly EditorTestWorkspace _workspace;
-    private readonly IThreadingContext _threadingContext;
     private readonly IAsynchronousOperationListenerProvider _listenerProvider;
 
     private AbstractDiagnosticsTaggerProvider<TTag>? _taggerProvider;
@@ -33,7 +31,6 @@ public DiagnosticTaggerWrapper(
         IReadOnlyDictionary<string, ImmutableArray<DiagnosticAnalyzer>>? analyzerMap = null,
         bool createTaggerProvider = true)
     {
-        _threadingContext = workspace.GetService<IThreadingContext>();
         _listenerProvider = workspace.GetService<IAsynchronousOperationListenerProvider>();
 
         var analyzerReference = new TestAnalyzerReferenceByLanguage(analyzerMap ?? DiagnosticExtensions.GetCompilerDiagnosticAnalyzersMap());

src/Features/Core/Portable/Copilot/CopilotChangeAnalysis.cs

@@ -15,10 +15,17 @@ namespace Microsoft.CodeAnalysis.Copilot;
 [DataContract]
 internal readonly record struct CopilotChangeAnalysis(
     [property: DataMember(Order = 0)] bool Succeeded,
-    [property: DataMember(Order = 1)] TimeSpan TotalAnalysisTime,
-    [property: DataMember(Order = 2)] TimeSpan TotalDiagnosticComputationTime,
-    [property: DataMember(Order = 3)] ImmutableArray<CopilotDiagnosticAnalysis> DiagnosticAnalyses,
-    [property: DataMember(Order = 4)] CopilotCodeFixAnalysis CodeFixAnalysis);
+    [property: DataMember(Order = 1)] int OldDocumentLength,
+    [property: DataMember(Order = 2)] int NewDocumentLength,
+    [property: DataMember(Order = 3)] int TextChangeDelta,
+    [property: DataMember(Order = 4)] int ProjectDocumentCount,
+    [property: DataMember(Order = 5)] int ProjectSourceGeneratedDocumentCount,
+    [property: DataMember(Order = 6)] int ProjectConeCount,
+    [property: DataMember(Order = 7)] TimeSpan TotalAnalysisTime,
+    [property: DataMember(Order = 8)] TimeSpan ForkingTime,
+    [property: DataMember(Order = 9)] TimeSpan TotalDiagnosticComputationTime,
+    [property: DataMember(Order = 10)] ImmutableArray<CopilotDiagnosticAnalysis> DiagnosticAnalyses,
+    [property: DataMember(Order = 11)] CopilotCodeFixAnalysis CodeFixAnalysis);
 
 /// <param name="Kind">What diagnostic kind this is analysis data for.</param>
 /// <param name="ComputationTime">How long it took to produce the diagnostics for this diagnostic kind.</param>

src/Features/Core/Portable/Copilot/CopilotChangeAnalysisUtilities.cs

@@ -0,0 +1,69 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Threading;
+using Microsoft.CodeAnalysis.Internal.Log;
+
+namespace Microsoft.CodeAnalysis.Copilot;
+
+internal static class CopilotChangeAnalysisUtilities
+{
+    public static IDisposable LogCopilotChangeAnalysis(
+        string featureId, bool accepted, string proposalId, CopilotChangeAnalysis analysisResult, CancellationToken cancellationToken)
+    {
+        return Logger.LogBlock(FunctionId.Copilot_AnalyzeChange, KeyValueLogMessage.Create(static (d, args) =>
+        {
+            var (featureId, accepted, proposalId, analysisResult) = args;
+            d["Accepted"] = accepted;
+            d["FeatureId"] = featureId;
+            d["ProposalId"] = proposalId;
+
+            d["Succeeded"] = analysisResult.Succeeded;
+
+            d["OldDocumentLength"] = analysisResult.OldDocumentLength;
+            d["NewDocumentLength"] = analysisResult.NewDocumentLength;
+            d["TextChangeDelta"] = analysisResult.TextChangeDelta;
+
+            d["ProjectDocumentCount"] = analysisResult.ProjectDocumentCount;
+            d["ProjectSourceGeneratedDocumentCount"] = analysisResult.ProjectSourceGeneratedDocumentCount;
+            d["ProjectConeCount"] = analysisResult.ProjectConeCount;
+
+            foreach (var diagnosticAnalysis in analysisResult.DiagnosticAnalyses)
+            {
+                var keyPrefix = $"DiagnosticAnalysis_{diagnosticAnalysis.Kind}";
+
+                d[$"{keyPrefix}_ComputationTime"] = Stringify(diagnosticAnalysis.ComputationTime);
+                d[$"{keyPrefix}_IdToCount"] = StringifyDictionary(diagnosticAnalysis.IdToCount);
+                d[$"{keyPrefix}_CategoryToCount"] = StringifyDictionary(diagnosticAnalysis.CategoryToCount);
+                d[$"{keyPrefix}_SeverityToCount"] = StringifyDictionary(diagnosticAnalysis.SeverityToCount);
+            }
+
+            d["CodeFixAnalysis_TotalComputationTime"] = Stringify(analysisResult.CodeFixAnalysis.TotalComputationTime);
+            d["CodeFixAnalysis_TotalApplicationTime"] = Stringify(analysisResult.CodeFixAnalysis.TotalApplicationTime);
+            d["CodeFixAnalysis_DiagnosticIdToCount"] = StringifyDictionary(analysisResult.CodeFixAnalysis.DiagnosticIdToCount);
+            d["CodeFixAnalysis_DiagnosticIdToApplicationTime"] = StringifyDictionary(analysisResult.CodeFixAnalysis.DiagnosticIdToApplicationTime);
+            d["CodeFixAnalysis_DiagnosticIdToProviderName"] = StringifyDictionary(analysisResult.CodeFixAnalysis.DiagnosticIdToProviderName);
+            d["CodeFixAnalysis_ProviderNameToApplicationTime"] = StringifyDictionary(analysisResult.CodeFixAnalysis.ProviderNameToApplicationTime);
+        }, args: (featureId, accepted, proposalId, analysisResult)),
+        cancellationToken);
+    }
+
+    private static string StringifyDictionary<TKey, TValue>(Dictionary<TKey, TValue> dictionary) where TKey : notnull where TValue : notnull
+        => string.Join(",", dictionary.Select(kvp => FormattableString.Invariant($"{kvp.Key}_{Stringify(kvp.Value)}")).OrderBy(v => v));
+
+    private static string Stringify<TValue>(TValue value) where TValue : notnull
+    {
+        if (value is IEnumerable<string> strings)
+            return string.Join(":", strings.OrderBy(v => v));
+
+        if (value is TimeSpan timeSpan)
+            return timeSpan.TotalMilliseconds.ToString("G17");
+
+        return value.ToString() ?? "";
+    }
+}

src/Features/Core/Portable/Copilot/CopilotOptions.cs

@@ -0,0 +1,12 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.CodeAnalysis.Options;
+
+namespace Microsoft.CodeAnalysis.Copilot;
+
+internal static class CopilotOptions
+{
+    public static Option2<bool> AnalyzeCopilotChanges { get; } = new Option2<bool>("dotnet_analyze_copilot_changes", defaultValue: true);
+}