From 0292fdf5beeef19d90dfa4f374c7a475e937899b Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Fri, 15 Jan 2021 13:26:04 -0800 Subject: [PATCH 01/12] Do not hold on to the entire StoreResult object in StoreResponseStatistics --- .../src/Diagnostics/CosmosDiagnosticsContextCore.cs | 4 ++-- .../Diagnostics/CosmosDiagnosticsSerializerVisitor.cs | 9 ++++++--- .../src/Diagnostics/StoreResponseStatistics.cs | 8 ++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs index 91d5f0f0c1..a98f9c97cb 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs @@ -137,9 +137,9 @@ internal override void AddDiagnosticsInternal(PointOperationStatistics pointOper internal override void AddDiagnosticsInternal(StoreResponseStatistics storeResponseStatistics) { - if (storeResponseStatistics.StoreResult != null) + if (storeResponseStatistics.StatusCode.HasValue) { - this.AddResponseCount((int)storeResponseStatistics.StoreResult.StatusCode); + this.AddResponseCount((int)storeResponseStatistics.StatusCode.Value); } this.AddToContextList(storeResponseStatistics); diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs index f036c5a907..612efbeb51 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs @@ -244,13 +244,16 @@ public override void Visit(StoreResponseStatistics storeResponseStatistics) this.jsonWriter.WritePropertyName("LocationEndpoint"); this.jsonWriter.WriteValue(storeResponseStatistics.LocationEndpoint); - if (storeResponseStatistics.StoreResult != null) + if (storeResponseStatistics.ActivityId != null) { this.jsonWriter.WritePropertyName("ActivityId"); - this.jsonWriter.WriteValue(storeResponseStatistics.StoreResult.ActivityId); + this.jsonWriter.WriteValue(storeResponseStatistics.ActivityId); + } + if (storeResponseStatistics.StoreResult != null) + { this.jsonWriter.WritePropertyName("StoreResult"); - this.jsonWriter.WriteValue(storeResponseStatistics.StoreResult.ToString()); + this.jsonWriter.WriteValue(storeResponseStatistics.StoreResult); } this.jsonWriter.WriteEndObject(); diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs index 057bbed19a..f92b80aff9 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs @@ -10,11 +10,13 @@ internal sealed class StoreResponseStatistics : CosmosDiagnosticsInternal { public readonly DateTime? RequestStartTime; public readonly DateTime RequestResponseTime; - public readonly StoreResult StoreResult; public readonly ResourceType RequestResourceType; public readonly OperationType RequestOperationType; public readonly Uri LocationEndpoint; public readonly bool IsSupplementalResponse; + public readonly StatusCodes? StatusCode; + public readonly string ActivityId; + public readonly string StoreResult; public StoreResponseStatistics( DateTime? requestStartTime, @@ -26,9 +28,11 @@ public StoreResponseStatistics( { this.RequestStartTime = requestStartTime; this.RequestResponseTime = requestResponseTime; - this.StoreResult = storeResult; this.RequestResourceType = resourceType; this.RequestOperationType = operationType; + this.StatusCode = storeResult?.StatusCode; + this.ActivityId = storeResult?.ActivityId; + this.StoreResult = storeResult?.ToString(); this.LocationEndpoint = locationEndpoint; this.IsSupplementalResponse = operationType == OperationType.Head || operationType == OperationType.HeadFeed; } From 82feb057cd57284a4c2292d7c4f674f60a051bf9 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Mon, 18 Jan 2021 19:14:32 -0800 Subject: [PATCH 02/12] Quick fix for unbounded memory consumption in cosmos diagnostics due to NetworkAttachedDocumentContainer hanging on to a single instance of diagnostics context and adding to it during long running operations --- .../src/Diagnostics/BoundedList.cs | 98 ++++++++++++++ .../src/Diagnostics/CircularQueue.cs | 128 ++++++++++++++++++ .../CosmosDiagnosticsContextCore.cs | 5 +- .../Diagnostics/BoundedListTests.cs | 49 +++++++ .../Diagnostics/CircularQueueTests.cs | 44 ++++++ 5 files changed, 322 insertions(+), 2 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs create mode 100644 Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs new file mode 100644 index 0000000000..1176e64d1e --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs @@ -0,0 +1,98 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.Diagnostics +{ + using System.Collections; + using System.Collections.Generic; + + /// + /// A list that can grow only up to a specified capacity. + /// In the growth phase, it uses the standard List type. + /// At capacity, it switches to a circular queue implementation. + /// + internal sealed class BoundedList : IEnumerable + { + private readonly int capacity; + + private List elementList; + + private CircularQueue circularQueue; + + private BoundedList(int capacity) + { + this.capacity = capacity; + this.elementList = new List(); + this.circularQueue = null; + } + + internal static bool TryCreate(int capacity, out BoundedList boundedList) + { + boundedList = null; + + if (capacity > 0) + { + boundedList = new BoundedList(capacity); + return true; + } + + return false; + } + + public void Add(T element) + { + if (this.circularQueue != null) + { + this.circularQueue.Add(element); + } + else if (this.elementList.Count < this.capacity) + { + this.elementList.Add(element); + } + else + { + _ = CircularQueue.TryCreate(this.capacity, out this.circularQueue); + this.circularQueue.AddRange(this.elementList); + this.elementList = null; + this.circularQueue.Add(element); + } + } + + public void AddRange(IEnumerable elements) + { + foreach (T element in elements) + { + this.Add(element); + } + } + + public IEnumerator GetListEnumerator() + { + // Using a for loop with a yield prevents Issue #1467 which causes + // ThrowInvalidOperationException if a new diagnostics is getting added + // while the enumerator is being used. + List elements = this.elementList; + for (int index = 0; index < elements.Count; ++index) + { + yield return elements[index]; + } + } + + public IEnumerator GetEnumerator() + { + if (this.circularQueue != null) + { + return this.circularQueue.GetEnumerator(); + } + else + { + return this.GetListEnumerator(); + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs new file mode 100644 index 0000000000..a4cbdbadb6 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs @@ -0,0 +1,128 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.Diagnostics +{ + using System; + using System.Collections; + using System.Collections.Generic; + + /// + /// simple circular queue that preallocates the underlying buffer + /// + internal sealed class CircularQueue : IEnumerable + { + private readonly T[] buffer; + + private int head; + private int tail; + + /// + /// Capacity of the queue. + /// + public int Capacity { get; } + + /// + /// True if adding an element will cause one to be evicted. + /// + public bool Full => this.Increment(this.tail) == this.head; + + /// + /// True when the queue is empty. + /// + public bool Empty => this.tail == this.head; + + /// + /// Initializes a new instance of the class. + /// + /// + private CircularQueue(int capacity) + { + this.head = 0; + this.tail = 0; + this.Capacity = capacity; + this.buffer = new T[capacity]; + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// the newly created queue + /// True if successful, false otherwise + public static bool TryCreate(int capacity, out CircularQueue queue) + { + queue = null; + if (capacity > 0) + { + queue = new CircularQueue(capacity + 1); + return true; + } + + return false; + } + + /// + /// Adds a new element to the queue. Can cause an older element to be evicted. + /// + /// + public void Add(T element) + { + if (this.Full) + { + this.TryPop(out _); + } + + this.buffer[this.tail] = element; + this.tail = this.Increment(this.tail); + } + + /// + /// Adds a subrange of the argument to the queue depending on capacity. + /// + /// + public void AddRange(IEnumerable elements) + { + foreach (T element in elements) + { + this.Add(element); + } + } + + private int Increment(int index) + { + return (index + 1) % this.Capacity; + } + + private bool TryPop(out T element) + { + element = default; + if (!this.Empty) + { + element = this.buffer[this.head]; + this.head = this.Increment(this.head); + return true; + } + + return false; + } + + /// + IEnumerator IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + + /// + public IEnumerator GetEnumerator() + { + if (!this.Empty) + { + for (int i = this.head; i != this.tail; i = this.Increment(i)) + { + yield return this.buffer[i]; + } + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs index a98f9c97cb..b6b8435537 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs @@ -23,7 +23,7 @@ internal sealed class CosmosDiagnosticsContextCore : CosmosDiagnosticsContext /// /// Detailed view of all the operations. /// - private List ContextList { get; } + private BoundedList ContextList { get; } private int totalResponseCount = 0; private int failedResponseCount = 0; @@ -49,7 +49,8 @@ public CosmosDiagnosticsContextCore( this.UserAgent = userAgentString ?? throw new ArgumentNullException(nameof(userAgentString)); this.OperationName = operationName ?? throw new ArgumentNullException(nameof(operationName)); this.StartUtc = DateTime.UtcNow; - this.ContextList = new List(); + _ = BoundedList.TryCreate(256, out BoundedList boundedList); + this.ContextList = boundedList; this.Diagnostics = new CosmosDiagnosticsCore(this); this.overallScope = new CosmosDiagnosticScope("Overall"); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs new file mode 100644 index 0000000000..683d88b52c --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs @@ -0,0 +1,49 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.Diagnostics +{ + using System.Linq; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class BoundedListTests + { + [TestMethod] + public void CapacityValidationTests() + { + foreach (int x in new[] {-512, -256, -1, 0 }) + { + Assert.IsFalse(BoundedList.TryCreate(x, out _)); + } + + foreach (int x in new[] { 1, 2, 8, 256 }) + { + Assert.IsTrue(BoundedList.TryCreate(x, out _)); + } + } + + [TestMethod] + [DataRow( 3, 21, DisplayName = "Extra small")] + [DataRow( 5, 25, DisplayName = "Small")] + [DataRow( 64, 256, DisplayName = "Medium")] + [DataRow( 256, 1024, DisplayName = "Large")] + [DataRow(2048, 4096, DisplayName = "Extra large")] + public void BasicTests(int capacity, int numElements) + { + Assert.IsTrue(BoundedList.TryCreate(capacity, out BoundedList boundedList)); + + for (int i = 0; i < numElements; ++i) + { + boundedList.Add(i); + + int expected = (i >= capacity) ? (i - capacity + 1) : 0; + foreach(int actual in boundedList) + { + Assert.AreEqual(expected, actual); + ++expected; + } + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs new file mode 100644 index 0000000000..e1d11cfee1 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs @@ -0,0 +1,44 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.Diagnostics +{ + using System.Linq; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CircularQueueTests + { + [TestMethod] + public void BasicTests() + { + Assert.IsTrue(CircularQueue.TryCreate(4, out CircularQueue queue)); + + Assert.IsTrue(queue.Empty); + Assert.IsFalse(queue.Full); + + queue.Add(1); + Assert.IsFalse(queue.Empty); + Assert.IsFalse(queue.Full); + + queue.Add(2); + queue.Add(3); + + int expected = 0; + foreach (int actual in queue) + { + Assert.AreEqual(++expected, actual); + } + + queue.Add(4); + Assert.IsTrue(queue.Full); + CollectionAssert.AreEquivalent(new[] { 1, 2, 3, 4 }, queue.ToArray()); + + queue.Add(5); + queue.Add(6); + Assert.IsTrue(queue.Full); + Assert.IsFalse(queue.Empty); + CollectionAssert.AreEquivalent(new[] { 3, 4, 5, 6 }, queue.ToList()); + } + } +} \ No newline at end of file From a0674d7721bb39e2823cb61d39bcfcb33d26a69c Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Mon, 18 Jan 2021 19:43:54 -0800 Subject: [PATCH 03/12] fix up bad merge --- .../src/Diagnostics/CosmosDiagnosticsContextCore.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs index b6b8435537..142e3b5160 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs @@ -200,9 +200,9 @@ public override IEnumerator GetEnumerator() // while the enumerator is being used. lock (this.ContextList) { - for (int i = 0; i < this.ContextList.Count; i++) + foreach (CosmosDiagnosticsInternal context in this.ContextList) { - yield return this.ContextList[i]; + yield return context; } } } From 2b50aa5b7ccfd545e2fee8f889936d4049207ac0 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Tue, 19 Jan 2021 12:33:12 -0800 Subject: [PATCH 04/12] fix up broken UT --- .../src/Diagnostics/CircularQueue.cs | 13 ++++++------- .../Diagnostics/CosmosDiagnosticsContextCore.cs | 3 ++- .../CosmosDiagnosticsUnitTests.cs | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs index a4cbdbadb6..cdb4ba6498 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs @@ -20,12 +20,12 @@ internal sealed class CircularQueue : IEnumerable /// /// Capacity of the queue. /// - public int Capacity { get; } + public int Capacity => this.buffer.Length; /// /// True if adding an element will cause one to be evicted. /// - public bool Full => this.Increment(this.tail) == this.head; + public bool Full => this.GetNextIndex(this.tail) == this.head; /// /// True when the queue is empty. @@ -40,7 +40,6 @@ private CircularQueue(int capacity) { this.head = 0; this.tail = 0; - this.Capacity = capacity; this.buffer = new T[capacity]; } @@ -74,7 +73,7 @@ public void Add(T element) } this.buffer[this.tail] = element; - this.tail = this.Increment(this.tail); + this.tail = this.GetNextIndex(this.tail); } /// @@ -89,7 +88,7 @@ public void AddRange(IEnumerable elements) } } - private int Increment(int index) + private int GetNextIndex(int index) { return (index + 1) % this.Capacity; } @@ -100,7 +99,7 @@ private bool TryPop(out T element) if (!this.Empty) { element = this.buffer[this.head]; - this.head = this.Increment(this.head); + this.head = this.GetNextIndex(this.head); return true; } @@ -118,7 +117,7 @@ public IEnumerator GetEnumerator() { if (!this.Empty) { - for (int i = this.head; i != this.tail; i = this.Increment(i)) + for (int i = this.head; i != this.tail; i = this.GetNextIndex(i)) { yield return this.buffer[i]; } diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs index 142e3b5160..99d8ee4fdd 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs @@ -18,6 +18,7 @@ namespace Microsoft.Azure.Cosmos internal sealed class CosmosDiagnosticsContextCore : CosmosDiagnosticsContext { private static readonly string DefaultUserAgentString; + internal static Func Capacity = () => 256; private readonly CosmosDiagnosticScope overallScope; /// @@ -49,7 +50,7 @@ public CosmosDiagnosticsContextCore( this.UserAgent = userAgentString ?? throw new ArgumentNullException(nameof(userAgentString)); this.OperationName = operationName ?? throw new ArgumentNullException(nameof(operationName)); this.StartUtc = DateTime.UtcNow; - _ = BoundedList.TryCreate(256, out BoundedList boundedList); + _ = BoundedList.TryCreate(Capacity(), out BoundedList boundedList); this.ContextList = boundedList; this.Diagnostics = new CosmosDiagnosticsCore(this); this.overallScope = new CosmosDiagnosticScope("Overall"); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs index 6887cd1c15..b8bedb4938 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs @@ -27,6 +27,12 @@ namespace Microsoft.Azure.Cosmos.Tests [TestClass] public class CosmosDiagnosticsUnitTests { + [TestCleanup] + public void Cleanup() + { + typeof(CosmosDiagnosticsContextCore).TypeInitializer.Invoke(null, null); + } + [TestMethod] public void ValidateActivityScope() { @@ -273,10 +279,12 @@ public void ValidateDiagnosticsAppendContext() } [TestMethod] - public void ValidateDiagnosticsAppendContextConcurrentCalls() + [DataRow(10, 100000, 1000001, DisplayName = "Unbounded")] + [DataRow(10, 100000, 256, DisplayName = "Bounded")] + public void ValidateDiagnosticsAppendContextConcurrentCalls(int threadCount, int itemCountPerThread, int expectedCount) { - int threadCount = 10; - int itemCountPerThread = 100000; + CosmosDiagnosticsContextCore.Capacity = () => expectedCount; + ConcurrentStack concurrentStack = new ConcurrentStack(); CosmosDiagnosticsContext cosmosDiagnostics = new CosmosDiagnosticsContextCore( nameof(ValidateDiagnosticsAppendContext), @@ -309,7 +317,7 @@ public void ValidateDiagnosticsAppendContextConcurrentCalls() Assert.AreEqual(0, concurrentStack.Count, $"Exceptions count: {concurrentStack.Count} Exceptions: {string.Join(';', concurrentStack)}"); int count = cosmosDiagnostics.Count(); - Assert.AreEqual((threadCount * itemCountPerThread) + 1, count); + Assert.AreEqual(expectedCount, count); } private void AddDiagnosticsInBackgroundLoop( From e9b2d073456b5aac91198a601f0600c7c9f2797c Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Wed, 20 Jan 2021 16:20:04 -0800 Subject: [PATCH 05/12] Incorporate code review feedback --- .../src/Diagnostics/BoundedList.cs | 23 ++++-------- .../src/Diagnostics/CircularQueue.cs | 36 +++++------------- .../CosmosDiagnosticsContextCore.cs | 5 +-- .../CosmosDiagnosticsUnitTests.cs | 37 ++++++++++--------- .../Diagnostics/BoundedListTests.cs | 17 +++++---- .../Diagnostics/CircularQueueTests.cs | 2 +- 6 files changed, 49 insertions(+), 71 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs index 1176e64d1e..df1a0370c0 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/BoundedList.cs @@ -3,6 +3,7 @@ //------------------------------------------------------------ namespace Microsoft.Azure.Cosmos.Diagnostics { + using System; using System.Collections; using System.Collections.Generic; @@ -19,24 +20,16 @@ internal sealed class BoundedList : IEnumerable private CircularQueue circularQueue; - private BoundedList(int capacity) + public BoundedList(int capacity) { - this.capacity = capacity; - this.elementList = new List(); - this.circularQueue = null; - } - - internal static bool TryCreate(int capacity, out BoundedList boundedList) - { - boundedList = null; - - if (capacity > 0) + if (capacity < 1) { - boundedList = new BoundedList(capacity); - return true; + throw new ArgumentOutOfRangeException("BoundedList capacity must be positive"); } - return false; + this.capacity = capacity; + this.elementList = new List(); + this.circularQueue = null; } public void Add(T element) @@ -51,7 +44,7 @@ public void Add(T element) } else { - _ = CircularQueue.TryCreate(this.capacity, out this.circularQueue); + this.circularQueue = new CircularQueue(this.capacity); this.circularQueue.AddRange(this.elementList); this.elementList = null; this.circularQueue.Add(element); diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs index cdb4ba6498..292bc3b872 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CircularQueue.cs @@ -36,29 +36,16 @@ internal sealed class CircularQueue : IEnumerable /// Initializes a new instance of the class. /// /// - private CircularQueue(int capacity) + public CircularQueue(int capacity) { - this.head = 0; - this.tail = 0; - this.buffer = new T[capacity]; - } - - /// - /// Initializes a new instance of the class. - /// - /// - /// the newly created queue - /// True if successful, false otherwise - public static bool TryCreate(int capacity, out CircularQueue queue) - { - queue = null; - if (capacity > 0) + if (capacity < 1) { - queue = new CircularQueue(capacity + 1); - return true; + throw new ArgumentOutOfRangeException("circular queue capacity must be positive"); } - return false; + this.head = 0; + this.tail = 0; + this.buffer = new T[capacity + 1]; // one empty slot } /// @@ -96,14 +83,11 @@ private int GetNextIndex(int index) private bool TryPop(out T element) { element = default; - if (!this.Empty) - { - element = this.buffer[this.head]; - this.head = this.GetNextIndex(this.head); - return true; - } + if (this.Empty) return false; - return false; + element = this.buffer[this.head]; + this.head = this.GetNextIndex(this.head); + return true; } /// diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs index 99d8ee4fdd..92c6826304 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs @@ -18,7 +18,7 @@ namespace Microsoft.Azure.Cosmos internal sealed class CosmosDiagnosticsContextCore : CosmosDiagnosticsContext { private static readonly string DefaultUserAgentString; - internal static Func Capacity = () => 256; + internal static Func Capacity = () => 5120; private readonly CosmosDiagnosticScope overallScope; /// @@ -50,8 +50,7 @@ public CosmosDiagnosticsContextCore( this.UserAgent = userAgentString ?? throw new ArgumentNullException(nameof(userAgentString)); this.OperationName = operationName ?? throw new ArgumentNullException(nameof(operationName)); this.StartUtc = DateTime.UtcNow; - _ = BoundedList.TryCreate(Capacity(), out BoundedList boundedList); - this.ContextList = boundedList; + this.ContextList = new BoundedList(Capacity()); this.Diagnostics = new CosmosDiagnosticsCore(this); this.overallScope = new CosmosDiagnosticScope("Overall"); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs index b8bedb4938..0e7c208f93 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs @@ -230,22 +230,6 @@ public void ValidateDiagnosticsAppendContext() using (cosmosDiagnostics.GetOverallScope()) { - // Test all the different operations on diagnostics context - using (cosmosDiagnostics.CreateScope("ValidateScope")) - { - Thread.Sleep(TimeSpan.FromSeconds(2)); - } - - cosmosDiagnostics2 = new CosmosDiagnosticsContextCore( - nameof(ValidateDiagnosticsAppendContext), - "MyCustomUserAgentString"); - cosmosDiagnostics2.GetOverallScope().Dispose(); - - using (cosmosDiagnostics.CreateScope("CosmosDiagnostics2Scope")) - { - Thread.Sleep(TimeSpan.FromMilliseconds(100)); - } - bool insertIntoDiagnostics1 = true; bool isInsertDiagnostics = false; // Start a background thread and ensure that no exception occurs even if items are getting added to the context @@ -257,13 +241,30 @@ public void ValidateDiagnosticsAppendContext() cpuLoadHistory: new Documents.Rntbd.CpuLoadHistory(new List().AsReadOnly(), TimeSpan.FromSeconds(1))); while (insertIntoDiagnostics1) { + Task.Delay(TimeSpan.FromMilliseconds(1)).Wait(); cosmosDiagnostics.AddDiagnosticsInternal(cosmosSystemInfo); } }); while (!isInsertDiagnostics) { - Task.Delay(TimeSpan.FromMilliseconds(10)).Wait(); + Task.Delay(TimeSpan.FromMilliseconds(5)).Wait(); + } + + // Test all the different operations on diagnostics context + using (cosmosDiagnostics.CreateScope("ValidateScope")) + { + Thread.Sleep(TimeSpan.FromMilliseconds(3)); + } + + cosmosDiagnostics2 = new CosmosDiagnosticsContextCore( + nameof(ValidateDiagnosticsAppendContext), + "MyCustomUserAgentString"); + cosmosDiagnostics2.GetOverallScope().Dispose(); + + using (cosmosDiagnostics.CreateScope("CosmosDiagnostics2Scope")) + { + Thread.Sleep(TimeSpan.FromMilliseconds(3)); } cosmosDiagnostics2.AddDiagnosticsInternal(cosmosDiagnostics); @@ -280,7 +281,7 @@ public void ValidateDiagnosticsAppendContext() [TestMethod] [DataRow(10, 100000, 1000001, DisplayName = "Unbounded")] - [DataRow(10, 100000, 256, DisplayName = "Bounded")] + [DataRow(10, 100000, 5120, DisplayName = "Bounded")] public void ValidateDiagnosticsAppendContextConcurrentCalls(int threadCount, int itemCountPerThread, int expectedCount) { CosmosDiagnosticsContextCore.Capacity = () => expectedCount; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs index 683d88b52c..12a81e3438 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/BoundedListTests.cs @@ -3,6 +3,7 @@ //------------------------------------------------------------ namespace Microsoft.Azure.Cosmos.Diagnostics { + using System; using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -14,24 +15,24 @@ public void CapacityValidationTests() { foreach (int x in new[] {-512, -256, -1, 0 }) { - Assert.IsFalse(BoundedList.TryCreate(x, out _)); + Assert.ThrowsException(() => new BoundedList(x)); } foreach (int x in new[] { 1, 2, 8, 256 }) { - Assert.IsTrue(BoundedList.TryCreate(x, out _)); + Assert.IsNotNull(new BoundedList(x)); } } [TestMethod] - [DataRow( 3, 21, DisplayName = "Extra small")] - [DataRow( 5, 25, DisplayName = "Small")] - [DataRow( 64, 256, DisplayName = "Medium")] - [DataRow( 256, 1024, DisplayName = "Large")] - [DataRow(2048, 4096, DisplayName = "Extra large")] + [DataRow( 3, 21, DisplayName = "Extra small")] + [DataRow( 5, 25, DisplayName = "Small")] + [DataRow( 256, 1024, DisplayName = "Medium")] + [DataRow( 5120, 10240, DisplayName = "Large")] + [DataRow(10240, 20480, DisplayName = "Large")] public void BasicTests(int capacity, int numElements) { - Assert.IsTrue(BoundedList.TryCreate(capacity, out BoundedList boundedList)); + BoundedList boundedList = new BoundedList(capacity); for (int i = 0; i < numElements; ++i) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs index e1d11cfee1..a25cec898b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/CircularQueueTests.cs @@ -12,7 +12,7 @@ public class CircularQueueTests [TestMethod] public void BasicTests() { - Assert.IsTrue(CircularQueue.TryCreate(4, out CircularQueue queue)); + CircularQueue queue = new CircularQueue(4); Assert.IsTrue(queue.Empty); Assert.IsFalse(queue.Full); From ddd72ff55a266a954c537a5158a5f4f4f74c0336 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Thu, 21 Jan 2021 11:33:40 -0800 Subject: [PATCH 06/12] Allocates the diagnostics string for StoreResult lazily rather than eagerly --- Directory.Build.props | 2 +- .../CosmosDiagnosticsContextCore.cs | 4 +- .../CosmosDiagnosticsSerializerVisitor.cs | 10 +-- .../Diagnostics/StoreResponseStatistics.cs | 23 +++-- .../src/Diagnostics/StoreResultStatistics.cs | 90 +++++++++++++++++++ .../Utils/DiagnosticValidators.cs | 2 +- 6 files changed, 114 insertions(+), 17 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs diff --git a/Directory.Build.props b/Directory.Build.props index aa4ee71715..7fb5d09cc8 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -3,7 +3,7 @@ 3.16.0 3.15.2 - 3.15.4 + 3.17.0 1.0.0-preview9 1.1.0-preview1 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs index 92c6826304..b18620fd0c 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs @@ -138,9 +138,9 @@ internal override void AddDiagnosticsInternal(PointOperationStatistics pointOper internal override void AddDiagnosticsInternal(StoreResponseStatistics storeResponseStatistics) { - if (storeResponseStatistics.StatusCode.HasValue) + if (storeResponseStatistics.StoreResultStatistics != null) { - this.AddResponseCount((int)storeResponseStatistics.StatusCode.Value); + this.AddResponseCount((int)storeResponseStatistics.StoreResultStatistics.StatusCode); } this.AddToContextList(storeResponseStatistics); diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs index 612efbeb51..f897f6f067 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsSerializerVisitor.cs @@ -244,16 +244,12 @@ public override void Visit(StoreResponseStatistics storeResponseStatistics) this.jsonWriter.WritePropertyName("LocationEndpoint"); this.jsonWriter.WriteValue(storeResponseStatistics.LocationEndpoint); - if (storeResponseStatistics.ActivityId != null) + if (storeResponseStatistics.StoreResultStatistics != null) { this.jsonWriter.WritePropertyName("ActivityId"); - this.jsonWriter.WriteValue(storeResponseStatistics.ActivityId); - } - - if (storeResponseStatistics.StoreResult != null) - { + this.jsonWriter.WriteValue(storeResponseStatistics.StoreResultStatistics.ActivityId); this.jsonWriter.WritePropertyName("StoreResult"); - this.jsonWriter.WriteValue(storeResponseStatistics.StoreResult); + this.jsonWriter.WriteValue(storeResponseStatistics.StoreResultStatistics.ToString()); } this.jsonWriter.WriteEndObject(); diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs index f92b80aff9..1c2e764b89 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs @@ -14,9 +14,7 @@ internal sealed class StoreResponseStatistics : CosmosDiagnosticsInternal public readonly OperationType RequestOperationType; public readonly Uri LocationEndpoint; public readonly bool IsSupplementalResponse; - public readonly StatusCodes? StatusCode; - public readonly string ActivityId; - public readonly string StoreResult; + public readonly StoreResultStatistics StoreResultStatistics; public StoreResponseStatistics( DateTime? requestStartTime, @@ -30,11 +28,24 @@ public StoreResponseStatistics( this.RequestResponseTime = requestResponseTime; this.RequestResourceType = resourceType; this.RequestOperationType = operationType; - this.StatusCode = storeResult?.StatusCode; - this.ActivityId = storeResult?.ActivityId; - this.StoreResult = storeResult?.ToString(); this.LocationEndpoint = locationEndpoint; this.IsSupplementalResponse = operationType == OperationType.Head || operationType == OperationType.HeadFeed; + + if (storeResult != null) + { + this.StoreResultStatistics = new StoreResultStatistics( + exception: storeResult.Exception, + partitionKeyRangeId: storeResult.PartitionKeyRangeId, + lsn: storeResult.LSN, + requestCharge: storeResult.RequestCharge, + isValid: storeResult.IsValid, + storePhysicalAddress: storeResult.StorePhysicalAddress, + globalCommittedLSN: storeResult.GlobalCommittedLSN, + itemLSN: storeResult.ItemLSN, + sessionToken: storeResult.SessionToken, + usingLocalLSN: storeResult.UsingLocalLSN, + activityId: storeResult.ActivityId); + } } public override void Accept(CosmosDiagnosticsInternalVisitor visitor) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs new file mode 100644 index 0000000000..003fa73f99 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs @@ -0,0 +1,90 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.Diagnostics +{ + using System; + using System.Globalization; + using System.Text; + using Microsoft.Azure.Documents; + + internal sealed class StoreResultStatistics + { + public DocumentClientException Exception { get; } + + public long LSN { get; } + + public string PartitionKeyRangeId { get; } + + public long GlobalCommittedLSN { get; } + + public long ItemLSN { get; } + + public ISessionToken SessionToken { get; } + + public bool UsingLocalLSN { get; } + + public bool IsValid { get; } + + public Uri StorePhysicalAddress { get; } + + public StatusCodes StatusCode { get; } + + public SubStatusCodes SubStatusCode { get; } + + public string ActivityId { get; } + + public double RequestCharge { get; } + + public StoreResultStatistics( + DocumentClientException exception, + string partitionKeyRangeId, + long lsn, + double requestCharge, + bool isValid, + Uri storePhysicalAddress, + long globalCommittedLSN, + long itemLSN, + ISessionToken sessionToken, + bool usingLocalLSN, + string activityId) + { + this.Exception = exception; + this.PartitionKeyRangeId = partitionKeyRangeId; + this.LSN = lsn; + this.RequestCharge = requestCharge; + this.IsValid = isValid; + this.StorePhysicalAddress = storePhysicalAddress; + this.GlobalCommittedLSN = globalCommittedLSN; + this.ItemLSN = itemLSN; + this.SessionToken = sessionToken; + this.UsingLocalLSN = usingLocalLSN; + this.ActivityId = activityId; + } + + public override string ToString() + { + StringBuilder stringBuilder = new StringBuilder(); + + stringBuilder.AppendFormat( + CultureInfo.InvariantCulture, + "StorePhysicalAddress: {0}, LSN: {1}, GlobalCommittedLsn: {2}, PartitionKeyRangeId: {3}, IsValid: {4}, StatusCode: {5}, SubStatusCode: {6}, " + + "RequestCharge: {7}, ItemLSN: {8}, SessionToken: {9}, UsingLocalLSN: {10}, TransportException: {11}", + this.StorePhysicalAddress, + this.LSN, + this.GlobalCommittedLSN, + this.PartitionKeyRangeId, + this.IsValid, + (int)this.StatusCode, + (int)this.SubStatusCode, + this.RequestCharge, + this.ItemLSN, + this.SessionToken?.ConvertToString(), + this.UsingLocalLSN, + this.Exception?.InnerException is TransportException ? this.Exception.InnerException.Message : "null"); + + return stringBuilder.ToString(); + } + + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/DiagnosticValidators.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/DiagnosticValidators.cs index 73e9c71f75..745a298961 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/DiagnosticValidators.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/DiagnosticValidators.cs @@ -185,7 +185,7 @@ private static void ValidateClientSideRequestStatistics(CosmosClientSideRequestS private static void ValidateStoreResponseStatistics(StoreResponseStatistics stats, DateTime startTimeUtc) { - Assert.IsNotNull(stats.StoreResult); + Assert.IsNotNull(stats.StoreResultStatistics); Assert.IsNotNull(stats.LocationEndpoint); Assert.IsTrue(startTimeUtc < stats.RequestResponseTime); Assert.IsTrue(stats.RequestResponseTime < DateTime.UtcNow); From e14ca49539c80adbd101239cfe7d550e86e6c640 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Thu, 21 Jan 2021 11:48:00 -0800 Subject: [PATCH 07/12] Add a test to check that the underlying response stream is freed --- .../Query/SanityQueryTests.cs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs index bafdd979eb..677c749cb9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs @@ -203,6 +203,56 @@ private static async Task> CreateWeakReferenceToFeedIterator return weakReferences; } + [TestMethod] + public async Task StoreResponseStatisticsMemoryLeak() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 100; + QueryOracleUtil util = new QueryOracle2(seed); + IEnumerable inputDocuments = util.GetDocuments(numberOfDocuments); + + await this.CreateIngestQueryDeleteAsync( + ConnectionModes.Direct, + CollectionTypes.MultiPartition, + inputDocuments, + ImplementationAsync); + + static async Task ImplementationAsync(Container container, IReadOnlyList documents) + { + using (FeedIterator feedIterator = container.GetItemQueryStreamIterator( + queryText: "SELECT * FROM c", + continuationToken: null, + requestOptions: new QueryRequestOptions + { + MaxItemCount = 10, + })) + { + WeakReference weakReference = await CreateWeakReferenceToResponseContent(feedIterator); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + await Task.Delay(500 /*ms*/); + + Assert.IsFalse(weakReference.IsAlive); + } + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static async Task CreateWeakReferenceToResponseContent( + FeedIterator feedIterator) + { + WeakReference weakResponseContent; + using (ResponseMessage response = await feedIterator.ReadNextAsync()) + { + Assert.IsNotNull(response.Content); + weakResponseContent = new WeakReference(response.Content, true); + } + + return weakResponseContent; + } + [TestMethod] public async Task TestNonDeterministicQueryResultsAsync() { From 745baef1133e6f24ed07a6008fa9c97903bcd7af Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Thu, 21 Jan 2021 11:50:52 -0800 Subject: [PATCH 08/12] more calls to force GC --- .../Query/SanityQueryTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs index 677c749cb9..38dc442793 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs @@ -233,6 +233,8 @@ static async Task ImplementationAsync(Container container, IReadOnlyList Date: Thu, 21 Jan 2021 12:46:31 -0800 Subject: [PATCH 09/12] Plumb Status and SubStatus correctly --- .../src/Diagnostics/StoreResponseStatistics.cs | 2 ++ .../src/Diagnostics/StoreResultStatistics.cs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs index 1c2e764b89..eb6d558c90 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResponseStatistics.cs @@ -35,6 +35,8 @@ public StoreResponseStatistics( { this.StoreResultStatistics = new StoreResultStatistics( exception: storeResult.Exception, + statusCode: storeResult.StatusCode, + subStatusCode: storeResult.SubStatusCode, partitionKeyRangeId: storeResult.PartitionKeyRangeId, lsn: storeResult.LSN, requestCharge: storeResult.RequestCharge, diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs index 003fa73f99..cc3bddc7e4 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/StoreResultStatistics.cs @@ -38,6 +38,8 @@ internal sealed class StoreResultStatistics public StoreResultStatistics( DocumentClientException exception, + StatusCodes statusCode, + SubStatusCodes subStatusCode, string partitionKeyRangeId, long lsn, double requestCharge, @@ -50,6 +52,8 @@ public StoreResultStatistics( string activityId) { this.Exception = exception; + this.StatusCode = statusCode; + this.SubStatusCode = subStatusCode; this.PartitionKeyRangeId = partitionKeyRangeId; this.LSN = lsn; this.RequestCharge = requestCharge; From d6789789fbee1d76203889fd4985ae1be7cec5fa Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Thu, 21 Jan 2021 12:55:58 -0800 Subject: [PATCH 10/12] Add region to keep in sync with LocationNames --- Microsoft.Azure.Cosmos/src/Regions.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Regions.cs b/Microsoft.Azure.Cosmos/src/Regions.cs index fdc1a16d03..7fe5845ba7 100644 --- a/Microsoft.Azure.Cosmos/src/Regions.cs +++ b/Microsoft.Azure.Cosmos/src/Regions.cs @@ -314,5 +314,10 @@ public static class Regions /// Name of the Azure Jio India West region in the Azure Cosmos DB service. /// public const string JioIndiaWest = "Jio India West"; + + /// + /// Name of the Azure US SLV region in the Azure Cosmos DB service. + /// + public const string EastUSSLV = "East US SLV"; } } From 684e1534fb2d13ee33104b4268ed82379a457494 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Thu, 21 Jan 2021 13:17:52 -0800 Subject: [PATCH 11/12] No need for CosmosDiagnosticsContextCore.Capacity to be a lambda --- .../src/Diagnostics/CosmosDiagnosticsContextCore.cs | 4 ++-- .../CosmosDiagnosticsUnitTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs index b18620fd0c..edcec0cf44 100644 --- a/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Diagnostics/CosmosDiagnosticsContextCore.cs @@ -18,7 +18,7 @@ namespace Microsoft.Azure.Cosmos internal sealed class CosmosDiagnosticsContextCore : CosmosDiagnosticsContext { private static readonly string DefaultUserAgentString; - internal static Func Capacity = () => 5120; + internal static int Capacity = 5120; private readonly CosmosDiagnosticScope overallScope; /// @@ -50,7 +50,7 @@ public CosmosDiagnosticsContextCore( this.UserAgent = userAgentString ?? throw new ArgumentNullException(nameof(userAgentString)); this.OperationName = operationName ?? throw new ArgumentNullException(nameof(operationName)); this.StartUtc = DateTime.UtcNow; - this.ContextList = new BoundedList(Capacity()); + this.ContextList = new BoundedList(Capacity); this.Diagnostics = new CosmosDiagnosticsCore(this); this.overallScope = new CosmosDiagnosticScope("Overall"); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs index 0e7c208f93..a66d2b9d19 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosDiagnosticsUnitTests.cs @@ -284,7 +284,7 @@ public void ValidateDiagnosticsAppendContext() [DataRow(10, 100000, 5120, DisplayName = "Bounded")] public void ValidateDiagnosticsAppendContextConcurrentCalls(int threadCount, int itemCountPerThread, int expectedCount) { - CosmosDiagnosticsContextCore.Capacity = () => expectedCount; + CosmosDiagnosticsContextCore.Capacity = expectedCount; ConcurrentStack concurrentStack = new ConcurrentStack(); CosmosDiagnosticsContext cosmosDiagnostics = new CosmosDiagnosticsContextCore( From 5bb16baef2f8330131c28f56581d62fd4dde0003 Mon Sep 17 00:00:00 2001 From: Neil Deshpande Date: Thu, 21 Jan 2021 14:07:45 -0800 Subject: [PATCH 12/12] update API to reflect the new region that was added --- .../Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json index 8be83578a9..e630e6b916 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json @@ -4596,6 +4596,11 @@ "Attributes": [], "MethodInfo": "System.String EastUS2EUAP;IsInitOnly:False;IsStatic:True;" }, + "System.String EastUSSLV": { + "Type": "Field", + "Attributes": [], + "MethodInfo": "System.String EastUSSLV;IsInitOnly:False;IsStatic:True;" + }, "System.String FranceCentral": { "Type": "Field", "Attributes": [],