Client Telemetry: Adds Thread Starvation Information

Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryHelper.cs

@@ -109,6 +109,8 @@ internal static void RecordSystemUsage(
             systemInfoCollection.Add(TelemetrySystemUsage.GetCpuInfo(systemUsageHistory.Values));
             systemInfoCollection.Add(TelemetrySystemUsage.GetMemoryRemainingInfo(systemUsageHistory.Values));
             systemInfoCollection.Add(TelemetrySystemUsage.GetAvailableThreadsInfo(systemUsageHistory.Values));
+            systemInfoCollection.Add(TelemetrySystemUsage.GetThreadWaitIntervalInMs(systemUsageHistory.Values));
+            systemInfoCollection.Add(TelemetrySystemUsage.GetIsThreadStarving(systemUsageHistory.Values));
         }
 
         /// <summary>

Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryOptions.cs

@@ -54,7 +54,17 @@ internal static class ClientTelemetryOptions
         internal const long AvailableThreadsMin = 1;
         internal const int AvailableThreadsPrecision = 2;
         internal const String AvailableThreadsName = "SystemPool_AvailableThreads";
-        internal const String AvailableThreadsUnit = "ThreadCount";
+        internal const String AvailableThreadsUnit = "ThreadCount";
+
+        // Expecting histogram to have Minimum ThreadWaitIntervalInMs of 1 and Maximum ThreadWaitIntervalInMs of 1 second
+        internal const long ThreadWaitIntervalInMsMax = TimeSpan.TicksPerSecond;
+        internal const long ThreadWaitIntervalInMsMin = 1;
+        internal const int ThreadWaitIntervalInMsPrecision = 2;
+        internal const string ThreadWaitIntervalInMsName = "SystemPool_ThreadWaitInterval";
+        internal const string ThreadWaitIntervalInMsUnit = "MilliSecond";
+
+        internal const string IsThreadStarvingName = "SystemPool_IsThreadStarving_True";
+        internal const string IsThreadStarvingUnit = "Count";
 
         internal const string DefaultVmMetadataUrL = "http://169.254.169.254/metadata/instance?api-version=2020-06-01";
         internal const double DefaultTimeStampInSeconds = 600;

Microsoft.Azure.Cosmos/src/Telemetry/MetricInfo.cs

@@ -21,11 +21,11 @@ internal MetricInfo(string metricsName, string unitName)
 
         public MetricInfo(string metricsName, 
             string unitName, 
-            double mean, 
-            long count, 
-            long min, 
-            long max, 
-            IReadOnlyDictionary<double, double> percentiles)
+            double mean = 0, 
+            long count = 0, 
+            long min = 0, 
+            long max = 0, 
+            IReadOnlyDictionary<double, double> percentiles = null)
             : this(metricsName, unitName)
         {
             this.Mean = mean;

Microsoft.Azure.Cosmos/src/Telemetry/SystemInfo.cs

@@ -22,6 +22,11 @@ internal SystemInfo(string metricsName, string unitName)
             this.MetricInfo = new MetricInfo(metricsName, unitName);
         }
 
+        internal SystemInfo(string metricsName, string unitName, int count)
+        {
+            this.MetricInfo = new MetricInfo(metricsName, unitName, count: count);
+        }
+
         public SystemInfo(MetricInfo metricInfo)
         {
             this.MetricInfo = metricInfo;

Microsoft.Azure.Cosmos/src/Telemetry/TelemetrySystemUsage.cs

@@ -4,6 +4,7 @@
 
 namespace Microsoft.Azure.Cosmos.Telemetry
 {
+    using System;
     using System.Collections.Generic;
     using HdrHistogram;
     using Microsoft.Azure.Documents.Rntbd;
@@ -35,7 +36,7 @@ public static SystemInfo GetCpuInfo(IReadOnlyCollection<SystemUsageLoad> systemU
                 long? infoToRecord = (long?)load.CpuUsage * ClientTelemetryOptions.HistogramPrecisionFactor;
                 if (infoToRecord.HasValue)
                 {
-                    histogram.RecordValue((long)infoToRecord);
+                    histogram.RecordValue(infoToRecord.Value);
                 }
             }
 
@@ -64,7 +65,7 @@ public static SystemInfo GetMemoryRemainingInfo(IReadOnlyCollection<SystemUsageL
                 long? infoToRecord = (long?)load.MemoryAvailable;
                 if (infoToRecord.HasValue)
                 {
-                    histogram.RecordValue((long)infoToRecord);
+                    histogram.RecordValue(infoToRecord.Value);
                 }
             }
 
@@ -93,7 +94,7 @@ public static SystemInfo GetAvailableThreadsInfo(IReadOnlyCollection<SystemUsage
                 long? infoToRecord = (long?)load.ThreadInfo?.AvailableThreads;
                 if (infoToRecord.HasValue)
                 {
-                    histogram.RecordValue((long)infoToRecord);
+                    histogram.RecordValue(infoToRecord.Value);
                 }
             }
 
@@ -105,5 +106,48 @@ public static SystemInfo GetAvailableThreadsInfo(IReadOnlyCollection<SystemUsage
             return systemInfo;
         }
 
+        public static SystemInfo GetIsThreadStarving(IReadOnlyCollection<SystemUsageLoad> systemUsageCollection)
+        {
+            int counter = 0;
+            foreach (SystemUsageLoad load in systemUsageCollection)
+            {
+                bool? infoToRecord = load.ThreadInfo?.IsThreadStarving;
+                if (infoToRecord.HasValue && infoToRecord.Value)
+                {
+                    counter++;
+                }
+            }
+            SystemInfo systemInfo = 
+                new SystemInfo(
+                    metricsName: ClientTelemetryOptions.IsThreadStarvingName, 
+                    unitName: ClientTelemetryOptions.IsThreadStarvingUnit,
+                    count: counter);
+
+            return systemInfo;
+        }
+
+        public static SystemInfo GetThreadWaitIntervalInMs(IReadOnlyCollection<SystemUsageLoad> systemUsageCollection)
+        {
+            LongConcurrentHistogram histogram = new LongConcurrentHistogram(ClientTelemetryOptions.ThreadWaitIntervalInMsMin,
+                                                        ClientTelemetryOptions.ThreadWaitIntervalInMsMax,
+                                                        ClientTelemetryOptions.ThreadWaitIntervalInMsPrecision);
+
+            SystemInfo systemInfo = new SystemInfo(ClientTelemetryOptions.ThreadWaitIntervalInMsName, ClientTelemetryOptions.ThreadWaitIntervalInMsUnit);
+            foreach (SystemUsageLoad load in systemUsageCollection)
+            {
+                double? infoToRecord = load.ThreadInfo?.ThreadWaitIntervalInMs;
+                if (infoToRecord.HasValue)
+                {
+                    histogram.RecordValue(TimeSpan.FromMilliseconds(infoToRecord.Value).Ticks);
+                }
+            }
+
+            if (histogram.TotalCount > 0)
+            {
+                systemInfo.SetAggregators(histogram);
+            }
+
+            return systemInfo;
+        }
     }
 }

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryTests.cs

@@ -765,7 +765,9 @@ private static void AssertSystemLevelInformation(List<SystemInfo> actualSystemIn
             {
                 { ClientTelemetryOptions.CpuName, ClientTelemetryOptions.CpuUnit },
                 { ClientTelemetryOptions.MemoryName, ClientTelemetryOptions.MemoryUnit },
-                { ClientTelemetryOptions.AvailableThreadsName, ClientTelemetryOptions.AvailableThreadsUnit }
+                { ClientTelemetryOptions.AvailableThreadsName, ClientTelemetryOptions.AvailableThreadsUnit },
+                { ClientTelemetryOptions.IsThreadStarvingName, ClientTelemetryOptions.IsThreadStarvingUnit },
+                { ClientTelemetryOptions.ThreadWaitIntervalInMsName, ClientTelemetryOptions.ThreadWaitIntervalInMsUnit }
             };
 
             Dictionary<string, string> actualMetricNameUnitMap = new Dictionary<string, string>();
@@ -851,7 +853,7 @@ private static void AssertAccountLevelInformation(List<ClientTelemetryProperties
                 actualOperationList.AddRange(telemetryInfo.OperationInfo);
                 actualSystemInformation.AddRange(telemetryInfo.SystemInfo);
 
-                Assert.AreEqual(3, telemetryInfo.SystemInfo.Count, $"System Information Count doesn't Match; {JsonConvert.SerializeObject(telemetryInfo.SystemInfo)}");
+                Assert.AreEqual(5, telemetryInfo.SystemInfo.Count, $"System Information Count doesn't Match; {JsonConvert.SerializeObject(telemetryInfo.SystemInfo)}");
 
                 Assert.IsNotNull(telemetryInfo.GlobalDatabaseAccountName, "GlobalDatabaseAccountName is null");
                 Assert.IsNotNull(telemetryInfo.DateTimeUtc, "Timestamp is null");