Diagnostics: Add CPU history

Microsoft.Azure.Cosmos/src/Diagnostics/BackendMetricsExtractor.cs

@@ -75,6 +75,11 @@ public override (ParseFailureReason, BackendMetrics) Visit(QueryPageDiagnostics
  return (ParseFailureReason.None, backendMetrics);
  }
 
+ public override (ParseFailureReason, BackendMetrics) Visit(CosmosSystemInfo cpuLoadHistory)
+ {
+ return BackendMetricsExtractor.MetricsNotFound;
+ }
+
  public override (ParseFailureReason, BackendMetrics) Visit(AddressResolutionStatistics addressResolutionStatistics)
  {
  return BackendMetricsExtractor.MetricsNotFound;

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContext.cs

@@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos
  using System.Collections;
  using System.Collections.Generic;
  using Microsoft.Azure.Cosmos.Diagnostics;
+ using Microsoft.Azure.Documents.Rntbd;
 
  /// <summary>
  /// This represents the diagnostics interface used in the SDK.
@@ -33,6 +34,8 @@ internal abstract class CosmosDiagnosticsContext : CosmosDiagnosticsInternal, IE
 
  internal abstract bool IsComplete();
 
+ internal abstract void AddDiagnosticsInternal(CosmosSystemInfo cpuLoadHistory);
+
  internal abstract void AddDiagnosticsInternal(PointOperationStatistics pointOperationStatistics);
 
  internal abstract void AddDiagnosticsInternal(QueryPageDiagnostics queryPageDiagnostics);

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs

@@ -9,6 +9,7 @@ namespace Microsoft.Azure.Cosmos
  using System.Diagnostics;
  using System.Linq;
  using Microsoft.Azure.Cosmos.Diagnostics;
+ using Microsoft.Azure.Documents.Rntbd;
 
  /// <summary>
  /// This represents the core diagnostics object used in the SDK.
@@ -84,6 +85,16 @@ internal override IDisposable CreateRequestHandlerScopeScope(RequestHandler requ
  return requestHandlerScope;
  }
 
+ internal override void AddDiagnosticsInternal(CosmosSystemInfo processInfo)
+ {
+ if (processInfo == null)
+ {
+ throw new ArgumentNullException(nameof(processInfo));
+ }
+
+ this.ContextList.Add(processInfo);
+ }
+
  internal override void AddDiagnosticsInternal(PointOperationStatistics pointOperationStatistics)
  {
  if (pointOperationStatistics == null)

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsInternalVisitor.cs

@@ -15,5 +15,6 @@ internal abstract class CosmosDiagnosticsInternalVisitor
  public abstract void Visit(StoreResponseStatistics storeResponseStatistics);
  public abstract void Visit(CosmosClientSideRequestStatistics clientSideRequestStatistics);
  public abstract void Visit(FeedRangeStatistics feedRangeStatistics);
+ public abstract void Visit(CosmosSystemInfo processInfo);
  }
 }

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsInternalVisitor{TResult}.cs

@@ -15,5 +15,6 @@ internal abstract class CosmosDiagnosticsInternalVisitor<TResult>
  public abstract TResult Visit(StoreResponseStatistics storeResponseStatistics);
  public abstract TResult Visit(CosmosClientSideRequestStatistics clientSideRequestStatistics);
  public abstract TResult Visit(FeedRangeStatistics feedRangeStatistics);
+ public abstract TResult Visit(CosmosSystemInfo processInfo);
  }
 }

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs

@@ -9,6 +9,7 @@ namespace Microsoft.Azure.Cosmos.Diagnostics
  using System.Globalization;
  using System.IO;
  using Microsoft.Azure.Documents;
+ using Microsoft.Azure.Documents.Rntbd;
  using Newtonsoft.Json;
 
  internal sealed class CosmosDiagnosticsSerializerVisitor : CosmosDiagnosticsInternalVisitor
@@ -281,6 +282,20 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
  this.jsonWriter.WriteEndObject();
  }
 
+ public override void Visit(CosmosSystemInfo processInfo)
+ {
+ this.jsonWriter.WriteStartObject();
+
+ this.jsonWriter.WritePropertyName("Id");
+ this.jsonWriter.WriteValue("SystemInfo");
+
+ this.jsonWriter.WritePropertyName("CpuHistory");
+ CpuLoadHistory cpuLoadHistory = processInfo.CpuLoadHistory;
+ this.jsonWriter.WriteValue(cpuLoadHistory.ToString());
+
+ this.jsonWriter.WriteEndObject();
+ }
+
  private void WriteJsonUriArray(string propertyName, IEnumerable<Uri> uris)
  {
  this.jsonWriter.WritePropertyName(propertyName);

Microsoft.Azure.Cosmos/src/Diagnostics/CosmosSystemInfo.cs

@@ -0,0 +1,29 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//------------------------------------------------------------
+namespace Microsoft.Azure.Cosmos.Diagnostics
+{
+ using System;
+ using Microsoft.Azure.Documents.Rntbd;
+
+ internal sealed class CosmosSystemInfo : CosmosDiagnosticsInternal
+ {
+ public readonly CpuLoadHistory CpuLoadHistory;
+
+ public CosmosSystemInfo(
+ CpuLoadHistory cpuLoadHistory)
+ {
+ this.CpuLoadHistory = cpuLoadHistory ?? throw new ArgumentNullException(nameof(cpuLoadHistory));
+ }
+
+ public override void Accept(CosmosDiagnosticsInternalVisitor visitor)
+ {
+ visitor.Visit(this);
+ }
+
+ public override TResult Accept<TResult>(CosmosDiagnosticsInternalVisitor<TResult> visitor)
+ {
+ return visitor.Visit(this);
+ }
+ }
+}

Microsoft.Azure.Cosmos/src/Diagnostics/EmptyCosmosDiagnosticsContext.cs

@@ -50,6 +50,10 @@ internal override IDisposable CreateRequestHandlerScopeScope(RequestHandler requ
  return EmptyCosmosDiagnosticsContext.DefaultScope;
  }
 
+ internal override void AddDiagnosticsInternal(CosmosSystemInfo cpuLoadHistory)
+ {
+ }
+
  internal override void AddDiagnosticsInternal(PointOperationStatistics pointOperationStatistics)
  {
  }

Microsoft.Azure.Cosmos/src/Handler/ClientPipelineBuilder.cs

@@ -14,6 +14,7 @@ internal class ClientPipelineBuilder
  {
  private readonly CosmosClient client;
  private readonly ConsistencyLevel? requestedClientConsistencyLevel;
+ private readonly DiagnosticsHandler diagnosticsHandler;
  private readonly RequestHandler invalidPartitionExceptionRetryHandler;
  private readonly RequestHandler transportHandler;
  private IReadOnlyCollection<RequestHandler> customHandlers;
@@ -35,6 +36,9 @@ public ClientPipelineBuilder(
  this.PartitionKeyRangeHandler = new PartitionKeyRangeHandler(client);
  Debug.Assert(this.PartitionKeyRangeHandler.InnerHandler == null, "The PartitionKeyRangeHandler.InnerHandler must be null to allow other handlers to be linked.");
 
+ this.diagnosticsHandler = new DiagnosticsHandler();
+ Debug.Assert(this.diagnosticsHandler.InnerHandler == null, nameof(this.diagnosticsHandler));
+
  this.UseRetryPolicy();
  this.AddCustomHandlers(customHandlers);
  }
@@ -129,6 +133,10 @@ public RequestInvokerHandler Build()
  }
  }
 
+ Debug.Assert(this.diagnosticsHandler != null, nameof(this.diagnosticsHandler));
+ current.InnerHandler = this.diagnosticsHandler;
+ current = current.InnerHandler;
+
  Debug.Assert(this.retryHandler != null, nameof(this.retryHandler));
  current.InnerHandler = this.retryHandler;
  current = current.InnerHandler;

Microsoft.Azure.Cosmos/src/Handler/DiagnosticsHandler.cs

@@ -0,0 +1,78 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Handlers
+{
+ using System;
+ using System.Threading;
+ using System.Threading.Tasks;
+ using Microsoft.Azure.Cosmos.Diagnostics;
+ using Microsoft.Azure.Documents.Rntbd;
+
+ /// <summary>
+ /// Handler which add process level info like CPU usage to the
+ /// diagnostics. This is a best effort scenario. It will not
+ /// add or attempt to add it if an exception occurs to avoid
+ /// impacting users.
+ /// </summary>
+ internal class DiagnosticsHandler : RequestHandler
+ {
+ public override Task<ResponseMessage> SendAsync(
+ RequestMessage request,
+ CancellationToken cancellationToken)
+ {
+ DiagnosticsHandlerHelper.Instance.RecordCpuDiagnostics(request);
+ return base.SendAsync(request, cancellationToken);
+ }
+
+ /// <summary>
+ /// This is a helper class that creates a single static instance to avoid each
+ /// client instance from creating a new CPU monitor.
+ /// </summary>
+ private class DiagnosticsHandlerHelper
+ {
+ public static readonly DiagnosticsHandlerHelper Instance = new DiagnosticsHandlerHelper();
+ private readonly CpuMonitor cpuMonitor = null;
+ private bool isCpuMonitorEnabled = false;
+
+ private DiagnosticsHandlerHelper()
+ {
+ // If the CPU monitor fails for some reason don't block the application
+ try
+ {
+ this.cpuMonitor = new CpuMonitor();
+ this.cpuMonitor.Start();
+ this.isCpuMonitorEnabled = true;
+ }
+ catch (Exception)
+ {
+ this.isCpuMonitorEnabled = false;
+ }
+ }
+
+ /// <summary>
+ /// The diagnostics should never block a request, and is a best attempt
+ /// If the CPU load history fails then don't try it in the future.
+ /// </summary>
+ public void RecordCpuDiagnostics(RequestMessage request)
+ {
+ if (this.isCpuMonitorEnabled)
+ {
+ try
+ {
+ CpuLoadHistory cpuHistory = this.cpuMonitor.GetCpuLoad();
+ if (cpuHistory != null)
+ {
+ request.DiagnosticsContext.AddDiagnosticsInternal(new CosmosSystemInfo(cpuHistory));
+ }
+ }
+ catch (Exception)
+ {
+ this.isCpuMonitorEnabled = false;
+ }
+ }
+ }
+ }
+ }
+}

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosDiagnosticsTests.cs

@@ -283,7 +283,12 @@ public async Task PointOperationDiagnostic(bool disableDiagnostics)
  if (disableDiagnostics)
  {
  requestOptions.DiagnosticContextFactory = () => EmptyCosmosDiagnosticsContext.Singleton;
- };
+ }
+ else
+ {
+ // Add 10 seconds to ensure CPU history is recorded
+ await Task.Delay(TimeSpan.FromSeconds(10));
+ }
 
  //Checking point operation diagnostics on typed operations
  ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/DiagnosticValidators.cs

@@ -88,6 +88,18 @@ private static void ValidateScope(CosmosDiagnosticScope scope, TimeSpan? totalEl
  Assert.IsTrue(elapsedInMs > 0);
  }
 
+ private static void ValidateProcessInfo(CosmosSystemInfo processInfo)
+ {
+ Assert.IsNotNull(processInfo.CpuLoadHistory);
+ Assert.IsNotNull(processInfo.CpuLoadHistory.ToString());
+
+ string info = processInfo.ToString();
+ Assert.IsNotNull(info);
+ JObject jObject = JObject.Parse(info);
+ Assert.AreEqual("SystemInfo", jObject["Id"].ToString());
+ Assert.IsNotNull(jObject["CpuHistory"].ToString());
+ }
+
  private static void ValidateRequestHandlerScope(RequestHandlerScope scope, TimeSpan? totalElapsedTime)
  {
  Assert.IsFalse(string.IsNullOrWhiteSpace(scope.Id));
@@ -291,6 +303,11 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
  // This will be visited if it is gateway query plan
  }
 
+ public override void Visit(CosmosSystemInfo cpuLoadHistory)
+ {
+ // This will be visited if it is gateway query plan
+ }
+
  public override void Visit(QueryPageDiagnostics queryPageDiagnostics)
  {
  this.isQueryPageVisited = true;
@@ -379,6 +396,12 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
  DiagnosticValidator.ValidateRequestHandlerScope(requestHandlerScope, this.TotalElapsedTime);
  }
 
+ public override void Visit(CosmosSystemInfo cpuLoadHistory)
+ {
+ Assert.IsTrue(this.isContextVisited);
+ DiagnosticValidator.ValidateProcessInfo(cpuLoadHistory);
+ }
+
  public override void Visit(CosmosDiagnosticScope cosmosDiagnosticScope)
  {
  }
@@ -471,6 +494,10 @@ public override void Visit(RequestHandlerScope requestHandlerScope)
  DiagnosticValidator.ValidateRequestHandlerScope(requestHandlerScope, this.TotalElapsedTime);
  }
 
+ public override void Visit(CosmosSystemInfo cpuLoadHistory)
+ {
+ }
+
  public override void Visit(CosmosDiagnosticScope cosmosDiagnosticScope)
  {
  }