Add metrics for datacollector.exe - provides information about profilers

scripts/build/TestPlatform.Dependencies.props

@@ -32,7 +32,7 @@
     <JsonNetVersion>9.0.1</JsonNetVersion>
     <MoqVersion>4.7.63</MoqVersion>
     <TestPlatformExternalsVersion>16.9.0-preview-4267359</TestPlatformExternalsVersion>
-    <CodeCoverageExternalsVersion>16.9.0-beta.21064.1</CodeCoverageExternalsVersion>
+    <CodeCoverageExternalsVersion>16.9.0-beta.21069.2</CodeCoverageExternalsVersion>
     <MicrosoftFakesVersion>16.9.0-beta.20628.1</MicrosoftFakesVersion>
 
     <MicrosoftBuildPackageVersion>16.0.461</MicrosoftBuildPackageVersion>

src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollection
+{
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using System.Collections.Generic;
+    using System.Collections.ObjectModel;
+    using System.Runtime.Serialization;
+
+    /// <summary>
+    /// Payload object that is used to exchange data between datacollector process and runner process.
+    /// </summary>
+    [DataContract]
+    public class AfterTestRunEndResult
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="AfterTestRunEndResult"/> class.
+        /// </summary>
+        /// <param name="environmentVariables">
+        /// The collection of attachment sets.
+        /// </param>
+        /// <param name="dataCollectionEventsPort">
+        /// The metrics.
+        /// </param>
+        public AfterTestRunEndResult(Collection<AttachmentSet> attachmentSets, IDictionary<string, object> metrics)
+        {
+            this.AttachmentSets = attachmentSets;
+            this.Metrics = metrics;
+        }
+
+        /// <summary>
+        /// Gets the environment variable dictionary.
+        /// </summary>
+        [DataMember]
+        public Collection<AttachmentSet> AttachmentSets { get; private set; }
+
+        /// <summary>
+        /// Get the Metrics
+        /// </summary>
+        [DataMember]
+        public IDictionary<string, object> Metrics { get; private set; }
+    }
+}

src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs

@@ -8,12 +8,13 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector
     using System.Collections.ObjectModel;
     using System.Globalization;
     using System.Linq;
-
     using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces;
     using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework;
     using Microsoft.VisualStudio.TestPlatform.Common.Logging;
+    using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
     using Microsoft.VisualStudio.TestPlatform.Common.Utilities;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
@@ -61,13 +62,18 @@ internal class DataCollectionManager : IDataCollectionManager
         /// </summary>
         private DataCollectorExtensionManager dataCollectorExtensionManager;
 
+        /// <summary>
+        /// Request data
+        /// </summary>
+        private IDataCollectionTelemetryManager dataCollectionTelemetryManager;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="DataCollectionManager"/> class.
         /// </summary>
         /// <param name="messageSink">
         /// The message Sink.
         /// </param>
-        internal DataCollectionManager(IMessageSink messageSink) : this(new DataCollectionAttachmentManager(), messageSink)
+        internal DataCollectionManager(IMessageSink messageSink, IRequestData requestData) : this(new DataCollectionAttachmentManager(), messageSink, new DataCollectionTelemetryManager(requestData))
         {
         }
 
@@ -83,13 +89,14 @@ internal DataCollectionManager(IMessageSink messageSink) : this(new DataCollecti
         /// <remarks>
         /// The constructor is not public because the factory method should be used to get instances of this class.
         /// </remarks>
-        protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink)
+        protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, IDataCollectionTelemetryManager dataCollectionTelemetryManager)
         {
             this.attachmentManager = datacollectionAttachmentManager;
             this.messageSink = messageSink;
             this.events = new TestPlatformDataCollectionEvents();
             this.dataCollectorExtensionManager = null;
             this.RunDataCollectors = new Dictionary<Type, DataCollectorInformation>();
+            this.dataCollectionTelemetryManager = dataCollectionTelemetryManager;
         }
 
         /// <summary>
@@ -136,7 +143,12 @@ public static DataCollectionManager Create(IMessageSink messageSink)
                 {
                     if (Instance == null)
                     {
-                        Instance = new DataCollectionManager(messageSink);
+                        var requestData = new RequestData
+                        {
+                            MetricsCollection = new MetricsCollection(),
+                            IsTelemetryOptedIn = true
+                        };
+                        Instance = new DataCollectionManager(messageSink, requestData);
                     }
                 }
             }
@@ -330,6 +342,8 @@ public Collection<AttachmentSet> TestCaseEnded(TestCaseEndEventArgs testCaseEndE
             return new Collection<AttachmentSet>(result);
         }
 
+        public IRequestData RequestData => this.dataCollectionTelemetryManager.RequestData;
+
         /// <summary>
         /// The dispose.
         /// </summary>
@@ -650,6 +664,7 @@ private void AddCollectorEnvironmentVariables(
                         if (string.Equals(namevaluepair.Value, alreadyRequestedVariable.Value, StringComparison.Ordinal))
                         {
                             alreadyRequestedVariable.AddRequestingDataCollector(collectorFriendlyName);
+                            dataCollectionTelemetryManager.OnEnvironmentVariableAdded(dataCollectionWrapper, namevaluepair.Key, namevaluepair.Value);
                         }
                         else
                         {
@@ -671,7 +686,9 @@ private void AddCollectorEnvironmentVariables(
                             else
                             {
                                 dataCollectionWrapper.Logger.LogError(this.dataCollectionEnvironmentContext.SessionDataCollectionContext, message);
-                            }                            
+                            }
+
+                            dataCollectionTelemetryManager.OnEnvironmentVariableConflict(dataCollectionWrapper, namevaluepair.Key, alreadyRequestedVariable.Value);
                         }
                     }
                     else
@@ -685,6 +702,7 @@ private void AddCollectorEnvironmentVariables(
                         dataCollectorEnvironmentVariables.Add(
                             namevaluepair.Key,
                             new DataCollectionEnvironmentVariable(namevaluepair, collectorFriendlyName));
+                        dataCollectionTelemetryManager.OnEnvironmentVariableAdded(dataCollectionWrapper, namevaluepair.Key, namevaluepair.Value);
                     }
                 }
             }

src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionTelemetryManager.cs

@@ -0,0 +1,107 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
+using System;
+using System.Linq;
+
+namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector
+{
+    /// <summary>
+    /// Stores and provides telemetry information for data collection.
+    /// </summary>
+    internal class DataCollectionTelemetryManager : IDataCollectionTelemetryManager
+    {
+        private const string CorProfilerVariable = "COR_PROFILER";
+        private const string CoreClrProfilerVariable = "CORECLR_PROFILER";
+        private const string ClrIeInstrumentationMethodConfigurationPrefix32Variable = "MicrosoftInstrumentationEngine_ConfigPath32_";
+        private const string ClrIeInstrumentationMethodConfigurationPrefix64Variable = "MicrosoftInstrumentationEngine_ConfigPath64_";
+
+        private const string VanguardProfilerGuid = "{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}";
+        private const string ClrIeProfilerGuid = "{324F817A-7420-4E6D-B3C1-143FBED6D855}";
+        private const string IntellitraceProfilerGuid = "{9317ae81-bcd8-47b7-aaa1-a28062e41c71}";
+
+        private const string VanguardProfilerName = "vanguard";
+        private const string ClrIeProfilerName = "clrie";
+        private const string IntellitraceProfilerName = "intellitrace";
+        private const string UnknownProfilerName = "unknown";
+        private const string OverwrittenProfilerName = "overwritten";
+
+        private const string CorProfilerTelemetryTemplate = "VS.TestPlatform.DataCollector.{0}.CorProfiler";
+        private const string CoreClrProfilerTelemetryTemplate = "VS.TestPlatform.DataCollector.{0}.CoreClrProfiler";
+
+        private readonly IRequestData requestData;
+
+        internal DataCollectionTelemetryManager(IRequestData requestData)
+        {
+            this.requestData = requestData;
+        }
+
+        /// <inheritdoc/>
+        public void OnEnvironmentVariableAdded(DataCollectorInformation dataCollectorInformation, string name, string value)
+        {
+            AddProfilerMetricForNewVariable(CorProfilerVariable, CorProfilerTelemetryTemplate, dataCollectorInformation, name, value);
+            AddProfilerMetricForNewVariable(CoreClrProfilerVariable, CoreClrProfilerTelemetryTemplate, dataCollectorInformation, name, value);
+        }
+
+        /// <inheritdoc/>
+        public void OnEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string existingValue)
+        {
+            AddProfilerMetricForConflictedVariable(CorProfilerVariable, CorProfilerTelemetryTemplate, dataCollectorInformation, name, existingValue);
+            AddProfilerMetricForConflictedVariable(CoreClrProfilerVariable, CoreClrProfilerTelemetryTemplate, dataCollectorInformation, name, existingValue);
+        }
+
+        /// <inheritdoc/>
+        public IRequestData RequestData => requestData;
+
+        private void AddProfilerMetricForNewVariable(string profilerVariable, string telemetryTemplateName, DataCollectorInformation dataCollectorInformation, string name, string value)
+        {
+            if (!string.Equals(profilerVariable, name, StringComparison.Ordinal))
+            {
+                return;
+            }
+
+            requestData.MetricsCollection.Add(string.Format(telemetryTemplateName, dataCollectorInformation.DataCollectorConfig?.TypeUri?.ToString()), GetProfilerName(value));
+        }
+
+        private void AddProfilerMetricForConflictedVariable(string profilerVariable, string telemetryTemplateName, DataCollectorInformation dataCollectorInformation, string name, string existingValue)
+        {
+            if (!string.Equals(profilerVariable, name, StringComparison.Ordinal))
+            {
+                return;
+            }
+
+            if (string.Equals(existingValue, ClrIeProfilerGuid, StringComparison.OrdinalIgnoreCase))
+            {
+                if (dataCollectorInformation.TestExecutionEnvironmentVariables != null &&
+                    dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix32Variable)) &&
+                    dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix64Variable)))
+                {
+                    requestData.MetricsCollection.Add(string.Format(telemetryTemplateName, dataCollectorInformation.DataCollectorConfig?.TypeUri?.ToString()), ClrIeProfilerName);
+                    return;
+                }
+            }
+
+            requestData.MetricsCollection.Add(string.Format(telemetryTemplateName, dataCollectorInformation.DataCollectorConfig?.TypeUri?.ToString()), OverwrittenProfilerName);
+        }
+
+        private static string GetProfilerName(string profilerGuid)
+        {
+            if (string.Equals(profilerGuid, VanguardProfilerGuid, StringComparison.OrdinalIgnoreCase))
+            {
+                return VanguardProfilerName;
+            }
+            else if (string.Equals(profilerGuid, ClrIeProfilerGuid, StringComparison.OrdinalIgnoreCase))
+            {
+                return ClrIeProfilerName;
+            }
+            else if (string.Equals(profilerGuid, IntellitraceProfilerGuid, StringComparison.OrdinalIgnoreCase))
+            {
+                return IntellitraceProfilerName;
+            }
+
+            return UnknownProfilerName;
+        }
+    }
+}

src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionManager.cs

@@ -8,6 +8,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces
     using System.Collections.ObjectModel;
 
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;
 
     /// <summary>
@@ -66,5 +67,13 @@ internal interface IDataCollectionManager : IDisposable
         /// Collection of session attachmentSet.
         /// </returns>
         Collection<AttachmentSet> SessionEnded(bool isCancelled);
+
+        /// <summary>
+        /// Provides telemetry object.
+        /// </summary>
+        /// <returns>
+        /// IRequestData object.
+        /// </returns>
+        IRequestData RequestData { get; }
     }
 }

src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionTelemetryManager.cs

@@ -0,0 +1,49 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
+
+namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces
+{
+    /// <summary>
+    /// The IDataCollectionTelemetryManager Interface.
+    /// </summary>
+    internal interface IDataCollectionTelemetryManager
+    {
+        /// <summary>
+        /// Stores telemetry regarding environment variable added.
+        /// </summary>
+        /// <param name="dataCollectorInformation">
+        /// Data collector information which requested environment variable.
+        /// </param>
+        /// <param name="name">
+        /// Environment variable name.
+        /// </param>
+        /// <param name="value">
+        /// Environment variable value.
+        /// </param>
+        void OnEnvironmentVariableAdded(DataCollectorInformation dataCollectorInformation, string name, string value);
+
+        /// <summary>
+        /// Stores telemetry regarding environment variable is conflicting.
+        /// </summary>
+        /// <param name="dataCollectorInformation">
+        /// Data collector information which requested environment variable.
+        /// </param>
+        /// <param name="name">
+        /// Environment variable name.
+        /// </param>
+        /// <param name="existingValue">
+        /// Environment variable value that was requested previously.
+        /// </param>
+        void OnEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string existingValue);
+
+        /// <summary>
+        /// Provides telemetry object.
+        /// </summary>
+        /// <returns>
+        /// IRequestData object.
+        /// </returns>
+        IRequestData RequestData { get; }
+    }
+}

src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs

@@ -21,6 +21,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollect
     using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces;
     using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
     using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
@@ -372,12 +373,13 @@ private void HandleAfterTestRunEnd(Message message)
             }
 
             var attachmentsets = this.dataCollectionManager.SessionEnded(isCancelled);
+            var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, this.dataCollectionManager.RequestData?.MetricsCollection?.Metrics);
 
             // Dispose all datacollectors before sending attachments to vstest.console process.
             // As datacollector process exits itself on parent process(vstest.console) exits.
             this.dataCollectionManager?.Dispose();
 
-            this.communicationManager.SendMessage(MessageType.AfterTestRunEndResult, attachmentsets);
+            this.communicationManager.SendMessage(MessageType.AfterTestRunEndResult, afterTestRunEndResult);
             EqtTrace.Info("DataCollectionRequestHandler.ProcessRequests : Session End message received from server. Closing the connection.");
 
             this.Close();

src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestSender.cs

@@ -4,7 +4,6 @@
 namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollection
 {
     using System.Collections.Generic;
-    using System.Collections.ObjectModel;
     using System.Globalization;
     using System.Net;
 
@@ -151,10 +150,10 @@ public BeforeTestRunStartResult SendBeforeTestRunStartAndGetResult(string settin
         }
 
         /// <inheritdoc/>
-        public Collection<AttachmentSet> SendAfterTestRunEndAndGetResult(ITestMessageEventHandler runEventsHandler, bool isCancelled)
+        public AfterTestRunEndResult SendAfterTestRunEndAndGetResult(ITestMessageEventHandler runEventsHandler, bool isCancelled)
         {
             var isDataCollectionComplete = false;
-            Collection<AttachmentSet> attachmentSets = null;
+            AfterTestRunEndResult result = null;
 
             if (EqtTrace.IsVerboseEnabled)
             {
@@ -181,12 +180,12 @@ public Collection<AttachmentSet> SendAfterTestRunEndAndGetResult(ITestMessageEve
                 }
                 else if (message.MessageType == MessageType.AfterTestRunEndResult)
                 {
-                    attachmentSets = this.dataSerializer.DeserializePayload<Collection<AttachmentSet>>(message);
+                    result = this.dataSerializer.DeserializePayload<AfterTestRunEndResult>(message);
                     isDataCollectionComplete = true;
                 }
             }
 
-            return attachmentSets;
+            return result;
         }
 
         private void LogDataCollectorMessage(DataCollectionMessageEventArgs dataCollectionMessageEventArgs, ITestMessageEventHandler requestHandler)