Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Direct version: Adds 3.19.3 #2502

Merged
merged 5 commits into from
May 26, 2021
Merged

Direct version: Adds 3.19.3 #2502

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d9d059e
Direct: Adds 3.19.3
May 26, 2021
28e5101
Merge branch 'master' into users/jawilley/direct/3.19.2
j82w May 26, 2021
1bd23bf
Add transport request stats to the diagnostics
May 26, 2021
b29a5d0
Merge branch 'users/jawilley/direct/3.19.2' of https://github.com/Azu…
May 26, 2021
14e775b
Updating it to actually have duration value.
May 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Directory.Build.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<ClientOfficialVersion>3.19.0</ClientOfficialVersion>
<ClientPreviewVersion>3.19.0</ClientPreviewVersion>
<ClientPreviewSuffixVersion>preview1</ClientPreviewSuffixVersion>
<DirectVersion>3.19.1</DirectVersion>
<DirectVersion>3.19.3</DirectVersion>
<EncryptionVersion>1.0.0-previewV15</EncryptionVersion>
<HybridRowVersion>1.1.0-preview3</HybridRowVersion>
<AboveDirBuildProps>$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))</AboveDirBuildProps>
Expand Down
80 changes: 62 additions & 18 deletions Microsoft.Azure.Cosmos/src/Tracing/TraceWriter.TraceJsonWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ public void Visit(PointOperationStatisticsTraceDatum pointOperationStatisticsTra
this.WriteStringValueOrNull(pointOperationStatisticsTraceDatum.ActivityId);

this.jsonWriter.WriteFieldName("ResponseTimeUtc");
this.jsonWriter.WriteStringValue(pointOperationStatisticsTraceDatum.ResponseTimeUtc.ToString("o", CultureInfo.InvariantCulture));
this.WriteDateTimeStringValue(pointOperationStatisticsTraceDatum.ResponseTimeUtc);

this.jsonWriter.WriteFieldName("StatusCode");
this.jsonWriter.WriteNumber64Value((int)pointOperationStatisticsTraceDatum.StatusCode);
Expand Down Expand Up @@ -215,7 +215,7 @@ public void Visit(ClientSideRequestStatisticsTraceDatum clientSideRequestStatist

foreach (ClientSideRequestStatisticsTraceDatum.AddressResolutionStatistics stat in clientSideRequestStatisticsTraceDatum.EndpointToAddressResolutionStatistics.Values)
{
VisitAddressResolutionStatistics(stat, this.jsonWriter);
this.VisitAddressResolutionStatistics(stat);
}

this.jsonWriter.WriteArrayEnd();
Expand Down Expand Up @@ -251,10 +251,10 @@ private void VisitHttpResponseStatistics(ClientSideRequestStatisticsTraceDatum.H
jsonWriter.WriteObjectStart();

jsonWriter.WriteFieldName("StartTimeUTC");
jsonWriter.WriteStringValue(stat.RequestStartTime.ToString("o", CultureInfo.InvariantCulture));
this.WriteDateTimeStringValue(stat.RequestStartTime);

jsonWriter.WriteFieldName("EndTimeUTC");
jsonWriter.WriteStringValue(stat.RequestEndTime.ToString("o", CultureInfo.InvariantCulture));
this.WriteDateTimeStringValue(stat.RequestEndTime);

jsonWriter.WriteFieldName("RequestUri");
jsonWriter.WriteStringValue(stat.RequestUri.ToString());
Expand Down Expand Up @@ -292,36 +292,35 @@ private void VisitHttpResponseStatistics(ClientSideRequestStatisticsTraceDatum.H
jsonWriter.WriteObjectEnd();
}

private static void VisitAddressResolutionStatistics(
ClientSideRequestStatisticsTraceDatum.AddressResolutionStatistics addressResolutionStatistics,
IJsonWriter jsonWriter)
private void VisitAddressResolutionStatistics(
ClientSideRequestStatisticsTraceDatum.AddressResolutionStatistics addressResolutionStatistics)
{
jsonWriter.WriteObjectStart();
this.jsonWriter.WriteObjectStart();

jsonWriter.WriteFieldName("StartTimeUTC");
jsonWriter.WriteStringValue(addressResolutionStatistics.StartTime.ToString("o", CultureInfo.InvariantCulture));
this.jsonWriter.WriteFieldName("StartTimeUTC");
this.WriteDateTimeStringValue(addressResolutionStatistics.StartTime);

jsonWriter.WriteFieldName("EndTimeUTC");
this.jsonWriter.WriteFieldName("EndTimeUTC");
if (addressResolutionStatistics.EndTime.HasValue)
{
jsonWriter.WriteStringValue(addressResolutionStatistics.EndTime.Value.ToString("o", CultureInfo.InvariantCulture));
this.WriteDateTimeStringValue(addressResolutionStatistics.EndTime.Value);
}
else
{
jsonWriter.WriteStringValue("EndTime Never Set.");
this.jsonWriter.WriteStringValue("EndTime Never Set.");
}

jsonWriter.WriteFieldName("TargetEndpoint");
this.jsonWriter.WriteFieldName("TargetEndpoint");
if (addressResolutionStatistics.TargetEndpoint == null)
{
jsonWriter.WriteNullValue();
this.jsonWriter.WriteNullValue();
}
else
{
jsonWriter.WriteStringValue(addressResolutionStatistics.TargetEndpoint);
this.jsonWriter.WriteStringValue(addressResolutionStatistics.TargetEndpoint);
}

jsonWriter.WriteObjectEnd();
this.jsonWriter.WriteObjectEnd();
}

private void VisitStoreResponseStatistics(
Expand All @@ -330,7 +329,7 @@ private void VisitStoreResponseStatistics(
this.jsonWriter.WriteObjectStart();

this.jsonWriter.WriteFieldName("ResponseTimeUTC");
this.jsonWriter.WriteStringValue(storeResponseStatistics.RequestResponseTime.ToString("o", CultureInfo.InvariantCulture));
this.WriteDateTimeStringValue(storeResponseStatistics.RequestResponseTime);

this.jsonWriter.WriteFieldName("ResourceType");
this.jsonWriter.WriteStringValue(storeResponseStatistics.RequestResourceType.ToString());
Expand Down Expand Up @@ -431,6 +430,8 @@ public void Visit(StoreResult storeResult)
this.jsonWriter.WriteFieldName("BELatencyInMs");
this.WriteStringValueOrNull(storeResult.BackendRequestDurationInMs);

this.VisitTransportRequestStats(storeResult.TransportRequestStats);

this.jsonWriter.WriteFieldName("TransportException");
TransportException transportException = storeResult.Exception?.InnerException as TransportException;
this.WriteStringValueOrNull(transportException?.Message);
Expand Down Expand Up @@ -470,6 +471,37 @@ private void WriteRegionsContactedArray(string propertyName, IEnumerable<(string
this.jsonWriter.WriteArrayEnd();
}

private void VisitTransportRequestStats(TransportRequestStats transportRequestStats)
{
this.jsonWriter.WriteFieldName("RntbdRequestStats");
if (transportRequestStats == null)
{
this.jsonWriter.WriteNullValue();
return;
}

IEnumerable<TransportRequestStats.RequestEvent> events = transportRequestStats.GetRequestTimeline();
this.jsonWriter.WriteArrayStart();

foreach (TransportRequestStats.RequestEvent requestEvent in events)
{
this.jsonWriter.WriteObjectStart();

this.jsonWriter.WriteFieldName("EventName");
this.jsonWriter.WriteStringValue(requestEvent.EventName);

this.jsonWriter.WriteFieldName("StartTime");
this.WriteDateTimeStringValue(requestEvent.StartTime);

this.jsonWriter.WriteFieldName("DurationInMicroSec");
this.jsonWriter.WriteStringValue(requestEvent.DurationInMicroSec.ToString(CultureInfo.InvariantCulture));

this.jsonWriter.WriteObjectEnd();
}

this.jsonWriter.WriteArrayEnd();
}

/// <summary>
/// Writes the list of URIs to JSON.
/// Sequential duplicates are counted and written as a single object to prevent
Expand Down Expand Up @@ -535,6 +567,18 @@ private void WriteStringValueOrNull(string value)
this.jsonWriter.WriteStringValue(value);
}
}

private void WriteDateTimeStringValue(DateTime value)
{
if (value == null)
{
this.jsonWriter.WriteNullValue();
}
else
{
this.jsonWriter.WriteStringValue(value.ToString("o", CultureInfo.InvariantCulture));
}
}
}
}
}
3 changes: 2 additions & 1 deletion Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/TransportWrapperTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,8 @@ private static void ThrowTransportExceptionOnItemOperation(
sessionToken: null,
usingLocalLSN: true,
activityId: Guid.NewGuid().ToString(),
backendRequestDurationInMs: "0"));
backendRequestDurationInMs: "0",
transportRequestStats: new TransportRequestStats()));

throw Documents.Rntbd.TransportExceptions.GetServiceUnavailableException(physicalAddress, Guid.NewGuid(),
transportException);
Expand Down
70 changes: 68 additions & 2 deletions ...osoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/TraceWriterBaselineTests.TraceData.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,8 @@
sessionToken: new SimpleSessionToken(42),
usingLocalLSN: true,
activityId: Guid.Empty.ToString(),
backendRequestDurationInMs: "4.2"),
backendRequestDurationInMs: "4.2",
transportRequestStats: TraceWriterBaselineTests.CreateTransportRequestStats()),
ResourceType.Document,
OperationType.Query,
uri1);
Expand Down Expand Up @@ -365,6 +366,38 @@
"StorePhysicalAddress": "http://storephysicaladdress.com/",
"RequestCharge": 3.14,
"BELatencyInMs": "4.2",
"RntbdRequestStats": [
{
"EventName": "Created",
"StartTime": "2021-12-31T23:59:59.059Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "ChannelAcquisitionStarted",
"StartTime": "2021-12-31T23:59:59.06Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Pipelined",
"StartTime": "2021-12-31T23:59:59.061Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Transit Time",
"StartTime": "2021-12-31T23:59:59.062Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Received",
"StartTime": "2021-12-31T23:59:59.063Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Completed",
"StartTime": "2021-12-31T23:59:59.064Z",
"DurationInMicroSec": "0"
}
],
"TransportException": null
}
}
Expand Down Expand Up @@ -417,7 +450,8 @@
sessionToken: default,
usingLocalLSN: default,
activityId: default,
backendRequestDurationInMs: default),
backendRequestDurationInMs: default,
transportRequestStats: TraceWriterBaselineTests.CreateTransportRequestStats()),
resourceType: default,
operationType: default,
locationEndpoint: default); ;
Expand Down Expand Up @@ -525,6 +559,38 @@
"StorePhysicalAddress": null,
"RequestCharge": 0,
"BELatencyInMs": null,
"RntbdRequestStats": [
{
"EventName": "Created",
"StartTime": "2021-12-31T23:59:59.059Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "ChannelAcquisitionStarted",
"StartTime": "2021-12-31T23:59:59.06Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Pipelined",
"StartTime": "2021-12-31T23:59:59.061Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Transit Time",
"StartTime": "2021-12-31T23:59:59.062Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Received",
"StartTime": "2021-12-31T23:59:59.063Z",
"DurationInMicroSec": "1000"
},
{
"EventName": "Completed",
"StartTime": "2021-12-31T23:59:59.064Z",
"DurationInMicroSec": "0"
}
],
"TransportException": null
}
}
Expand Down
5 changes: 4 additions & 1 deletion Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Tracing/TraceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ public void ValidateStoreResultSerialization()
sessionToken: new SimpleSessionToken(42),
usingLocalLSN: true,
activityId: Guid.Empty.ToString(),
backendRequestDurationInMs: "4.2");
backendRequestDurationInMs: "4.2",
transportRequestStats: TraceWriterBaselineTests.CreateTransportRequestStats());

StoreResponseStatistics storeResponseStatistics = new StoreResponseStatistics(
DateTime.MinValue,
Expand All @@ -138,6 +139,8 @@ public void ValidateStoreResultSerialization()
storeResultProperties.Remove(nameof(storeResult.BackendRequestDurationInMs));
storeResultProperties.Add("TransportException");
storeResultProperties.Remove(nameof(storeResult.Exception));
storeResultProperties.Add("RntbdRequestStats");
storeResultProperties.Remove(nameof(storeResult.TransportRequestStats));

foreach (string key in jsonPropertyNames)
{
Expand Down
51 changes: 49 additions & 2 deletions ...osoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Tracing/TraceWriterBaselineTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ namespace Microsoft.Azure.Cosmos.Tests.Tracing
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using System.Xml;
Expand Down Expand Up @@ -400,7 +401,8 @@ public void TraceData()
sessionToken: new SimpleSessionToken(42),
usingLocalLSN: true,
activityId: Guid.Empty.ToString(),
backendRequestDurationInMs: "4.2"),
backendRequestDurationInMs: "4.2",
transportRequestStats: TraceWriterBaselineTests.CreateTransportRequestStats()),
ResourceType.Document,
OperationType.Query,
uri1);
Expand Down Expand Up @@ -453,7 +455,8 @@ public void TraceData()
sessionToken: default,
usingLocalLSN: default,
activityId: default,
backendRequestDurationInMs: default),
backendRequestDurationInMs: default,
transportRequestStats: TraceWriterBaselineTests.CreateTransportRequestStats()),
resourceType: default,
operationType: default,
locationEndpoint: default); ;
Expand Down Expand Up @@ -708,6 +711,50 @@ private static async Task<IDocumentContainer> CreateDocumentContainerAsync(
return documentContainer;
}

internal static TransportRequestStats CreateTransportRequestStats()
{
DateTime defaultDateTime = new DateTime(
year: 2021,
month: 12,
day: 31,
hour: 23,
minute: 59,
second: 59,
millisecond: 59,
kind: DateTimeKind.Utc);

TransportRequestStats transportRequestStats = new TransportRequestStats();
transportRequestStats.RecordState(TransportRequestStats.RequestStage.ChannelAcquisitionStarted);
transportRequestStats.RecordState(TransportRequestStats.RequestStage.Pipelined);
transportRequestStats.RecordState(TransportRequestStats.RequestStage.Sent);
transportRequestStats.RecordState(TransportRequestStats.RequestStage.Received);
transportRequestStats.RecordState(TransportRequestStats.RequestStage.Completed);

FieldInfo field = transportRequestStats.GetType().GetField("requestCreatedTime", BindingFlags.NonPublic | BindingFlags.Instance);
field.SetValue(transportRequestStats, defaultDateTime);

defaultDateTime += TimeSpan.FromMilliseconds(1);
field = transportRequestStats.GetType().GetField("channelAcquisitionStartedTime", BindingFlags.NonPublic | BindingFlags.Instance);
field.SetValue(transportRequestStats, defaultDateTime);

defaultDateTime += TimeSpan.FromMilliseconds(1);
field = transportRequestStats.GetType().GetField("requestPipelinedTime", BindingFlags.NonPublic | BindingFlags.Instance);
field.SetValue(transportRequestStats, defaultDateTime);

defaultDateTime += TimeSpan.FromMilliseconds(1);
field = transportRequestStats.GetType().GetField("requestSentTime", BindingFlags.NonPublic | BindingFlags.Instance);
field.SetValue(transportRequestStats, defaultDateTime);

defaultDateTime += TimeSpan.FromMilliseconds(1);
field = transportRequestStats.GetType().GetField("requestReceivedTime", BindingFlags.NonPublic | BindingFlags.Instance);
field.SetValue(transportRequestStats, defaultDateTime);

defaultDateTime += TimeSpan.FromMilliseconds(1);
field = transportRequestStats.GetType().GetField("requestCompletedTime", BindingFlags.NonPublic | BindingFlags.Instance);
field.SetValue(transportRequestStats, defaultDateTime);
return transportRequestStats;
}

private static IQueryPipelineStage CreatePipeline(IDocumentContainer documentContainer, string query, int pageSize = 10, CosmosElement state = null)
{
TryCatch<IQueryPipelineStage> tryCreatePipeline = PipelineFactory.MonadicCreate(
Expand Down