Diagnostics: Add CPU history (#1602)

* Diagnostics: Add CPU history to all diagnostics
Azure · Jun 11, 2020 · 180ef23 · 180ef23
1 parent 04db851
commit 180ef23
Show file tree

Hide file tree

Showing 13 changed files with 568 additions and 357 deletions.
diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/BackendMetricsExtractor.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/BackendMetricsExtractor.cs
@@ -75,6 +75,11 @@ public override (ParseFailureReason, BackendMetrics) Visit(QueryPageDiagnostics
             return (ParseFailureReason.None, backendMetrics);
         }
 
+        public override (ParseFailureReason, BackendMetrics) Visit(CosmosSystemInfo cpuLoadHistory)
+        {
+            return BackendMetricsExtractor.MetricsNotFound;
+        }
+
         public override (ParseFailureReason, BackendMetrics) Visit(AddressResolutionStatistics addressResolutionStatistics)
         {
             return BackendMetricsExtractor.MetricsNotFound;

diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContext.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContext.cs
@@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos
     using System.Collections;
     using System.Collections.Generic;
     using Microsoft.Azure.Cosmos.Diagnostics;
+    using Microsoft.Azure.Documents.Rntbd;
 
     /// <summary>
     /// This represents the diagnostics interface used in the SDK.
@@ -33,6 +34,8 @@ internal abstract class CosmosDiagnosticsContext : CosmosDiagnosticsInternal, IE
 
         internal abstract bool IsComplete();
 
+        internal abstract void AddDiagnosticsInternal(CosmosSystemInfo cpuLoadHistory);
+
         internal abstract void AddDiagnosticsInternal(PointOperationStatistics pointOperationStatistics);
 
         internal abstract void AddDiagnosticsInternal(QueryPageDiagnostics queryPageDiagnostics);

diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs
@@ -9,6 +9,7 @@ namespace Microsoft.Azure.Cosmos
     using System.Diagnostics;
     using System.Linq;
     using Microsoft.Azure.Cosmos.Diagnostics;
+    using Microsoft.Azure.Documents.Rntbd;
 
     /// <summary>
     /// This represents the core diagnostics object used in the SDK.
@@ -84,6 +85,16 @@ internal override IDisposable CreateRequestHandlerScopeScope(RequestHandler requ
             return requestHandlerScope;
         }
 
+        internal override void AddDiagnosticsInternal(CosmosSystemInfo processInfo)
+        {
+            if (processInfo == null)
+            {
+                throw new ArgumentNullException(nameof(processInfo));
+            }
+
+            this.ContextList.Add(processInfo);
+        }
+
         internal override void AddDiagnosticsInternal(PointOperationStatistics pointOperationStatistics)
         {
             if (pointOperationStatistics == null)

diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsInternalVisitor.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsInternalVisitor.cs
@@ -15,5 +15,6 @@ internal abstract class CosmosDiagnosticsInternalVisitor
         public abstract void Visit(StoreResponseStatistics storeResponseStatistics);
         public abstract void Visit(CosmosClientSideRequestStatistics clientSideRequestStatistics);
         public abstract void Visit(FeedRangeStatistics feedRangeStatistics);
+        public abstract void Visit(CosmosSystemInfo processInfo);
     }
 }
diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsInternalVisitor{TResult}.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsInternalVisitor{TResult}.cs
@@ -15,5 +15,6 @@ internal abstract class CosmosDiagnosticsInternalVisitor<TResult>
         public abstract TResult Visit(StoreResponseStatistics storeResponseStatistics);
         public abstract TResult Visit(CosmosClientSideRequestStatistics clientSideRequestStatistics);
         public abstract TResult Visit(FeedRangeStatistics feedRangeStatistics);
+        public abstract TResult Visit(CosmosSystemInfo processInfo);
     }
 }
diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs
@@ -9,6 +9,7 @@ namespace Microsoft.Azure.Cosmos.Diagnostics
     using System.Globalization;
     using System.IO;
     using Microsoft.Azure.Documents;
+    using Microsoft.Azure.Documents.Rntbd;
     using Newtonsoft.Json;
 
     internal sealed class CosmosDiagnosticsSerializerVisitor : CosmosDiagnosticsInternalVisitor
@@ -281,6 +282,20 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
             this.jsonWriter.WriteEndObject();
         }
 
+        public override void Visit(CosmosSystemInfo processInfo)
+        {
+            this.jsonWriter.WriteStartObject();
+
+            this.jsonWriter.WritePropertyName("Id");
+            this.jsonWriter.WriteValue("SystemInfo");
+
+            this.jsonWriter.WritePropertyName("CpuHistory");
+            CpuLoadHistory cpuLoadHistory = processInfo.CpuLoadHistory;
+            this.jsonWriter.WriteValue(cpuLoadHistory.ToString());
+
+            this.jsonWriter.WriteEndObject();
+        }
+
         private void WriteJsonUriArray(string propertyName, IEnumerable<Uri> uris)
         {
             this.jsonWriter.WritePropertyName(propertyName);

diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosSystemInfo.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosSystemInfo.cs
@@ -0,0 +1,29 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+namespace Microsoft.Azure.Cosmos.Diagnostics
+{
+    using System;
+    using Microsoft.Azure.Documents.Rntbd;
+
+    internal sealed class CosmosSystemInfo : CosmosDiagnosticsInternal
+    {
+        public readonly CpuLoadHistory CpuLoadHistory;
+
+        public CosmosSystemInfo(
+            CpuLoadHistory cpuLoadHistory)
+        {
+            this.CpuLoadHistory = cpuLoadHistory ?? throw new ArgumentNullException(nameof(cpuLoadHistory));
+        }
+
+        public override void Accept(CosmosDiagnosticsInternalVisitor visitor)
+        {
+            visitor.Visit(this);
+        }
+
+        public override TResult Accept<TResult>(CosmosDiagnosticsInternalVisitor<TResult> visitor)
+        {
+            return visitor.Visit(this);
+        }
+    }
+}
diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/EmptyCosmosDiagnosticsContext.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/EmptyCosmosDiagnosticsContext.cs
@@ -50,6 +50,10 @@ internal override IDisposable CreateRequestHandlerScopeScope(RequestHandler requ
             return EmptyCosmosDiagnosticsContext.DefaultScope;
         }
 
+        internal override void AddDiagnosticsInternal(CosmosSystemInfo cpuLoadHistory)
+        {
+        }
+
         internal override void AddDiagnosticsInternal(PointOperationStatistics pointOperationStatistics)
         {
         }

diff --git a/Microsoft.Azure.Cosmos/src/Handler/ClientPipelineBuilder.cs b/Microsoft.Azure.Cosmos/src/Handler/ClientPipelineBuilder.cs
@@ -14,6 +14,7 @@ internal class ClientPipelineBuilder
     {
         private readonly CosmosClient client;
         private readonly ConsistencyLevel? requestedClientConsistencyLevel;
+        private readonly DiagnosticsHandler diagnosticsHandler;
         private readonly RequestHandler invalidPartitionExceptionRetryHandler;
         private readonly RequestHandler transportHandler;
         private IReadOnlyCollection<RequestHandler> customHandlers;
@@ -35,6 +36,9 @@ public ClientPipelineBuilder(
             this.PartitionKeyRangeHandler = new PartitionKeyRangeHandler(client);
             Debug.Assert(this.PartitionKeyRangeHandler.InnerHandler == null, "The PartitionKeyRangeHandler.InnerHandler must be null to allow other handlers to be linked.");
 
+            this.diagnosticsHandler = new DiagnosticsHandler();
+            Debug.Assert(this.diagnosticsHandler.InnerHandler == null, nameof(this.diagnosticsHandler));
+
             this.UseRetryPolicy();
             this.AddCustomHandlers(customHandlers);
         }
@@ -129,6 +133,10 @@ public RequestInvokerHandler Build()
                 }
             }
 
+            Debug.Assert(this.diagnosticsHandler != null, nameof(this.diagnosticsHandler));
+            current.InnerHandler = this.diagnosticsHandler;
+            current = current.InnerHandler;
+
             Debug.Assert(this.retryHandler != null, nameof(this.retryHandler));
             current.InnerHandler = this.retryHandler;
             current = current.InnerHandler;

diff --git a/Microsoft.Azure.Cosmos/src/Handler/DiagnosticsHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/DiagnosticsHandler.cs
@@ -0,0 +1,78 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Handlers
+{
+    using System;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Microsoft.Azure.Cosmos.Diagnostics;
+    using Microsoft.Azure.Documents.Rntbd;
+
+    /// <summary>
+    /// Handler which add process level info like CPU usage to the
+    /// diagnostics. This is a best effort scenario. It will not
+    /// add or attempt to add it if an exception occurs to avoid
+    /// impacting users.
+    /// </summary>
+    internal class DiagnosticsHandler : RequestHandler
+    {
+        public override Task<ResponseMessage> SendAsync(
+            RequestMessage request,
+            CancellationToken cancellationToken)
+        {
+            DiagnosticsHandlerHelper.Instance.RecordCpuDiagnostics(request);
+            return base.SendAsync(request, cancellationToken);
+        }
+
+        /// <summary>
+        /// This is a helper class that creates a single static instance to avoid each
+        /// client instance from creating a new CPU monitor.
+        /// </summary>
+        private class DiagnosticsHandlerHelper
+        {
+            public static readonly DiagnosticsHandlerHelper Instance = new DiagnosticsHandlerHelper();
+            private readonly CpuMonitor cpuMonitor = null;
+            private bool isCpuMonitorEnabled = false;
+
+            private DiagnosticsHandlerHelper()
+            {
+                // If the CPU monitor fails for some reason don't block the application
+                try
+                {
+                    this.cpuMonitor = new CpuMonitor();
+                    this.cpuMonitor.Start();
+                    this.isCpuMonitorEnabled = true;
+                }
+                catch (Exception)
+                {
+                    this.isCpuMonitorEnabled = false;
+                }
+            }
+
+            /// <summary>
+            /// The diagnostics should never block a request, and is a best attempt
+            /// If the CPU load history fails then don't try it in the future.
+            /// </summary>
+            public void RecordCpuDiagnostics(RequestMessage request)
+            {
+                if (this.isCpuMonitorEnabled)
+                {
+                    try
+                    {
+                        CpuLoadHistory cpuHistory = this.cpuMonitor.GetCpuLoad();
+                        if (cpuHistory != null)
+                        {
+                            request.DiagnosticsContext.AddDiagnosticsInternal(new CosmosSystemInfo(cpuHistory));
+                        }
+                    }
+                    catch (Exception)
+                    {
+                        this.isCpuMonitorEnabled = false;
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosDiagnosticsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosDiagnosticsTests.cs
@@ -283,7 +283,12 @@ public async Task PointOperationDiagnostic(bool disableDiagnostics)
             if (disableDiagnostics)
             {
                 requestOptions.DiagnosticContextFactory = () => EmptyCosmosDiagnosticsContext.Singleton;
-            };
+            }
+            else
+            {
+                // Add 10 seconds to ensure CPU history is recorded
+                await Task.Delay(TimeSpan.FromSeconds(10));
+            }
 
             //Checking point operation diagnostics on typed operations
             ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();

diff --git a/...oft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/DiagnosticValidators.cs b/...oft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/DiagnosticValidators.cs
@@ -88,6 +88,18 @@ private static void ValidateScope(CosmosDiagnosticScope scope, TimeSpan? totalEl
             Assert.IsTrue(elapsedInMs > 0);
         }
 
+        private static void ValidateProcessInfo(CosmosSystemInfo processInfo)
+        {
+            Assert.IsNotNull(processInfo.CpuLoadHistory);
+            Assert.IsNotNull(processInfo.CpuLoadHistory.ToString());
+
+            string info = processInfo.ToString();
+            Assert.IsNotNull(info);
+            JObject jObject = JObject.Parse(info);
+            Assert.AreEqual("SystemInfo", jObject["Id"].ToString());
+            Assert.IsNotNull(jObject["CpuHistory"].ToString());
+        }
+
         private static void ValidateRequestHandlerScope(RequestHandlerScope scope, TimeSpan? totalElapsedTime)
         {
             Assert.IsFalse(string.IsNullOrWhiteSpace(scope.Id));
@@ -291,6 +303,11 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
                // This will be visited if it is gateway query plan
             }
 
+            public override void Visit(CosmosSystemInfo cpuLoadHistory)
+            {
+                // This will be visited if it is gateway query plan
+            }
+
             public override void Visit(QueryPageDiagnostics queryPageDiagnostics)
             {
                 this.isQueryPageVisited = true;
@@ -379,6 +396,12 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
                 DiagnosticValidator.ValidateRequestHandlerScope(requestHandlerScope, this.TotalElapsedTime);
             }
 
+            public override void Visit(CosmosSystemInfo cpuLoadHistory)
+            {
+                Assert.IsTrue(this.isContextVisited);
+                DiagnosticValidator.ValidateProcessInfo(cpuLoadHistory);
+            }
+
             public override void Visit(CosmosDiagnosticScope cosmosDiagnosticScope)
             {
             }
@@ -471,6 +494,10 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
                 DiagnosticValidator.ValidateRequestHandlerScope(requestHandlerScope, this.TotalElapsedTime);
             }
 
+            public override void Visit(CosmosSystemInfo cpuLoadHistory)
+            {
+            }
+
             public override void Visit(CosmosDiagnosticScope cosmosDiagnosticScope)
             {
             }