[LiveMetrics] report process metrics CPU Total and Committed Memory

sdk/monitor/Azure.Monitor.OpenTelemetry.AspNetCore/src/Azure.Monitor.OpenTelemetry.AspNetCore.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <Description>An OpenTelemetry .NET distro that exports to Azure Monitor</Description>
     <AssemblyTitle>AzureMonitor OpenTelemetry ASP.NET Core Distro</AssemblyTitle>
@@ -24,11 +24,11 @@
 
     <!-- FOR PUBLIC RELEASES, MUST USE PackageReference. THIS REQUIRES A STAGGERED RELEASE IF SHIPPING A NEW EXPORTER. -->
     <PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" />
-    <PackageReference Include="Azure.Monitor.OpenTelemetry.LiveMetrics" />
+    <!--<PackageReference Include="Azure.Monitor.OpenTelemetry.LiveMetrics" />-->
 
     <!-- FOR LOCAL DEV, ProjectReference IS PREFERRED. -->
     <!--<ProjectReference Include="..\..\Azure.Monitor.OpenTelemetry.Exporter\src\Azure.Monitor.OpenTelemetry.Exporter.csproj" />-->
-    <!--<ProjectReference Include="..\..\Azure.Monitor.OpenTelemetry.LiveMetrics\src\Azure.Monitor.OpenTelemetry.LiveMetrics.csproj" />-->
+    <ProjectReference Include="..\..\Azure.Monitor.OpenTelemetry.LiveMetrics\src\Azure.Monitor.OpenTelemetry.LiveMetrics.csproj" />
   </ItemGroup>
 
   <!-- Shared source from Exporter -->

sdk/monitor/Azure.Monitor.OpenTelemetry.AspNetCore/tests/Azure.Monitor.OpenTelemetry.AspNetCore.Demo/Program.cs

@@ -2,16 +2,21 @@
 // Licensed under the MIT License.
 
 #if NET6_0_OR_GREATER
+using System;
 using System.Diagnostics;
 using System.Net.Http;
+using System.Threading.Tasks;
+using System.Threading;
 using Azure.Monitor.OpenTelemetry.AspNetCore;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 
+CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
+
 var builder = WebApplication.CreateBuilder(args);
 
-builder.Services.AddOpenTelemetry().UseAzureMonitor();
+//builder.Services.AddOpenTelemetry().UseAzureMonitor();
 
 /*
 builder.Services.AddOpenTelemetry().UseAzureMonitor(o =>
@@ -33,5 +38,52 @@
     return $"Hello World! OpenTelemetry Trace: {Activity.Current?.Id}";
 });
 
+#if !NETFRAMEWORK
+app.MapGet("/StressTest", () =>
+{
+    RunStressTest();
+    return "StressTest!";
+});
+#endif
+
 app.Run();
 #endif
+
+#if !NETFRAMEWORK
+void RunStressTest()
+{
+    // Get the number of available processors
+    int numProcessors = Environment.ProcessorCount;
+
+    Task[] tasks = new Task[numProcessors];
+
+    // Start each task
+    for (int i = 0; i < numProcessors; i++)
+    {
+        tasks[i] = Task.Run(() => Compute(cancellationTokenSource.Token));
+    }
+
+    var timeStamp = DateTime.Now.AddSeconds(20);
+    while (DateTime.Now < timeStamp)
+    {
+        // do nothing
+    }
+
+    cancellationTokenSource.Cancel();
+
+    Task.WaitAll(tasks);
+}
+
+void Compute(CancellationToken cancellationToken)
+{
+    while (!cancellationToken.IsCancellationRequested)
+    {
+        // Simulate intensive computation
+        double result = 0;
+        for (int i = 0; i < 1000000; i++)
+        {
+            result += Math.Sqrt(i);
+        }
+    }
+}
+#endif

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Features Added
 
+- Report Performance Counters "CPU Total" and "Committed Memory".
+  ([#42213](https://github.com/Azure/azure-sdk-for-net/pull/42213))
+
 ### Breaking Changes
 
 ### Bugs Fixed

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Diagnostics/LiveMetricsExporterEventSource.cs

@@ -195,5 +195,14 @@ public void DroppedDocument(DocumentIngressDocumentType documentType)
 
         [Event(12, Message = "Document was dropped. DocumentType: {0}. Not user actionable.", Level = EventLevel.Warning)]
         public void DroppedDocument(string documentType) => WriteEvent(12, documentType);
+
+        [Event(13, Message = "Failure to calculate CPU Counter. Unexpected negative timespan: PreviousCollectedTime: {0}. RecentCollectedTime: {0}. Not user actionable.", Level = EventLevel.Error)]
+        public void ProcessCountersUnexpectedNegativeTimeSpan(long previousCollectedTime, long recentCollectedTime) => WriteEvent(13, previousCollectedTime, recentCollectedTime);
+
+        [Event(14, Message = "Failure to calculate CPU Counter. Unexpected negative value: PreviousCollectedValue: {0}. RecentCollectedValue: {0}. Not user actionable.", Level = EventLevel.Error)]
+        public void ProcessCountersUnexpectedNegativeValue(long previousCollectedValue, long recentCollectedValue) => WriteEvent(14, previousCollectedValue, recentCollectedValue);
+
+        [Event(15, Message = "Calculated Cpu Counter: Period: {0}. DiffValue: {1}. CalculatedValue: {2}. ProcessorCount: {3}. NormalizedValue: {4}", Level = EventLevel.Verbose)]
+        public void ProcessCountersCpuCounter(long period, long diffValue, double calculatedValue, int processorCount, double normalizedValue) => WriteEvent(15, period, diffValue, calculatedValue, processorCount, normalizedValue);
     }
 }

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.Metrics.cs

@@ -18,8 +18,8 @@ internal partial class Manager
         internal readonly DoubleBuffer _documentBuffer = new();
         internal static bool? s_isAzureWebApp = null;
 
-        //private readonly PerformanceCounter _performanceCounter_ProcessorTime = new(categoryName: "Processor", counterName: "% Processor Time", instanceName: "_Total");
-        //private readonly PerformanceCounter _performanceCounter_CommittedBytes = new(categoryName: "Memory", counterName: "Committed Bytes");
+        private DateTimeOffset cachedCollectedTime = DateTimeOffset.MinValue;
+        private long cachedCollectedValue = 0;
 
         public MonitoringDataPoint GetDataPoint()
         {
@@ -91,32 +91,47 @@ public MonitoringDataPoint GetDataPoint()
                 dataPoint.Metrics.Add(metricPoint);
             }
 
-            // TODO: Reenable Perf Counters
-            //foreach (var metricPoint in CollectPerfCounters())
-            //{
-            //    dataPoint.Metrics.Add(metricPoint);
-            //}
+            foreach (var metricPoint in CollectPerfCounters())
+            {
+                dataPoint.Metrics.Add(metricPoint);
+            }
 
             return dataPoint;
         }
 
-        //public IEnumerable<Models.MetricPoint> CollectPerfCounters()
-        //{
-        //    // PERFORMANCE COUNTERS
-        //    yield return new Models.MetricPoint
-        //    {
-        //        Name = LiveMetricConstants.MetricId.MemoryCommittedBytesMetricIdValue,
-        //        Value = _performanceCounter_CommittedBytes.NextValue(),
-        //        Weight = 1
-        //    };
-
-        //    yield return new Models.MetricPoint
-        //    {
-        //        Name = LiveMetricConstants.MetricId.ProcessorTimeMetricIdValue,
-        //        Value = _performanceCounter_ProcessorTime.NextValue(),
-        //        Weight = 1
-        //    };
-        //}
+        /// <summary>
+        /// Collect Perf Counters for the current process.
+        /// </summary>
+        /// <remarks>
+        /// For Memory:
+        /// <see href="https://learn.microsoft.com/dotnet/api/system.diagnostics.process.privatememorysize64"/>.
+        /// "The amount of memory, in bytes, allocated for the associated process that cannot be shared with other processes.".
+        ///
+        /// For CPU:
+        /// <see href="https://learn.microsoft.com/dotnet/api/system.diagnostics.process.totalprocessortime"/>.
+        /// "A TimeSpan that indicates the amount of time that the associated process has spent utilizing the CPU. This value is the sum of the UserProcessorTime and the PrivilegedProcessorTime.".
+        /// </remarks>
+        public IEnumerable<Models.MetricPoint> CollectPerfCounters()
+        {
+            var process = Process.GetCurrentProcess();
+
+            yield return new Models.MetricPoint
+            {
+                Name = LiveMetricConstants.MetricId.MemoryCommittedBytesMetricIdValue,
+                Value = process.PrivateMemorySize64,
+                Weight = 1
+            };
+
+            if (TryCalculateCPUCounter(process, out var processorValue))
+            {
+                yield return new Models.MetricPoint
+                {
+                    Name = LiveMetricConstants.MetricId.ProcessorTimeMetricIdValue,
+                    Value = Convert.ToSingle(processorValue),
+                    Weight = 1
+                };
+            }
+        }
 
         /// <summary>
         /// Searches for the environment variable specific to Azure Web App.
@@ -149,5 +164,70 @@ public MonitoringDataPoint GetDataPoint()
 
             return s_isAzureWebApp;
         }
+
+        private void ResetCachedValues()
+        {
+            this.cachedCollectedTime = DateTimeOffset.MinValue;
+            this.cachedCollectedValue = 0;
+        }
+
+        /// <summary>
+        /// Calcualte the CPU usage as the diff between two ticks divided by the period of time, and then divided by the number of processors.
+        /// </summary>
+        private bool TryCalculateCPUCounter(Process process, out double normalizedValue)
+        {
+            var previousCollectedValue = this.cachedCollectedValue;
+            var previousCollectedTime = this.cachedCollectedTime;
+
+            var recentCollectedValue = this.cachedCollectedValue = process.TotalProcessorTime.Ticks;
+            var recentCollectedTime = this.cachedCollectedTime = DateTimeOffset.UtcNow;
+
+            var processorCount = Environment.ProcessorCount;
+
+            double calculatedValue;
+
+            if (previousCollectedTime == DateTimeOffset.MinValue)
+            {
+                Debug.WriteLine($"{nameof(TryCalculateCPUCounter)} DateTimeOffset.MinValue");
+                normalizedValue = default;
+                return false;
+            }
+
+            var period = recentCollectedTime.Ticks - previousCollectedTime.Ticks;
+            if (period < 0)
+            {
+                // Not likely to happen but being safe here incase of clock issues in multi-core.
+                LiveMetricsExporterEventSource.Log.ProcessCountersUnexpectedNegativeTimeSpan(
+                    previousCollectedTime: previousCollectedTime.Ticks,
+                    recentCollectedTime: recentCollectedTime.Ticks);
+                Debug.WriteLine($"{nameof(TryCalculateCPUCounter)} period less than zero");
+                normalizedValue = default;
+                return false;
+            }
+
+            var diff = recentCollectedValue - previousCollectedValue;
+            if (diff < 0)
+            {
+                LiveMetricsExporterEventSource.Log.ProcessCountersUnexpectedNegativeValue(
+                    previousCollectedValue: previousCollectedValue,
+                    recentCollectedValue: recentCollectedValue);
+                Debug.WriteLine($"{nameof(TryCalculateCPUCounter)} diff less than zero");
+                normalizedValue = default;
+                return false;
+            }
+
+            period = period != 0 ? period : 1;
+            calculatedValue = diff * 100.0 / period;
+            normalizedValue = calculatedValue / processorCount;
+            LiveMetricsExporterEventSource.Log.ProcessCountersCpuCounter(
+                period: previousCollectedValue,
+                diffValue: recentCollectedValue,
+                calculatedValue: calculatedValue,
+                processorCount: processorCount,
+                normalizedValue: normalizedValue);
+            // TryCalculateCPUCounter period: 10313304 diff: 64062500 calculatedValue: 621.1636930318354 processorCount: 8 normalizedValue: 77.64546162897942
+            Debug.WriteLine($"{nameof(TryCalculateCPUCounter)} period: {period} diff: {diff} calculatedValue: {calculatedValue} processorCount: {processorCount} normalizedValue: {normalizedValue}");
+            return true;
+        }
     }
 }

sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.State.cs

@@ -71,6 +71,9 @@ private void SetPingState()
             // This is used in determining if we should Backoff.
             // If we've been in another state for X amount of time, that may exceed our maximum interval and immediately trigger a Backoff.
             _lastSuccessfulPing = DateTimeOffset.UtcNow;
+
+            // Must reset the metrics cache here.
+            ResetCachedValues();
         }
 
         private void SetPostState()