Skip to content
This repository has been archived by the owner on Jul 30, 2024. It is now read-only.
/ NuGet.Jobs Public archive

China CDN log fix + minor global CDN log fix #948

Merged
merged 7 commits into from
Jan 12, 2021
Merged

China CDN log fix + minor global CDN log fix #948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c573958
Tests update
agr Dec 8, 2020
0a11308
Fix
agr Dec 8, 2020
f0a171e
No trailing space for "global" logs.
agr Dec 8, 2020
303ae20
Less string concatenation.
agr Dec 8, 2020
d35a8c5
Update src/Stats.AzureCdnLogs.Common/Collect/OutputLogLine.cs
agr Dec 9, 2020
076eeb6
`.Append` on its own line.
agr Dec 9, 2020
1eeef04
Line number as a named constant.
agr Dec 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/Stats.AzureCdnLogs.Common/Collect/OutputLogLine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,22 @@ public static string Header

public override string ToString()
{
return $"{TimeStamp} {TimeTaken} {CIp} {FileSize} {SIp} {SPort} {ScStatus} {ScBytes} {CsMethod} {CsUriStem} - {RsDuration} {RsBytes} {CReferrer} {CUserAgent} {CustomerId} {XEc_Custom_1}";
return $"{TimeStamp} {TimeTaken} {CIp} {FileSize} {SIp} {SPort} {ScStatus} {ScBytes} {CsMethod} {CsUriStem} - {RsDuration} {RsBytes} {CReferrer} {Quote(CUserAgent)} {CustomerId} {Quote(XEc_Custom_1)}";
}

private static string Quote(string input)
{
if (input.StartsWith("\"") && input.EndsWith("\""))
{
// already quoted
return input;
}
if (input.IndexOfAny(new[] { ' ', '\t' }) >= 0)
{
return $"\"{input}\"";
}

return input;
}
}
}
67 changes: 50 additions & 17 deletions src/Stats.CollectAzureCdnLogs/Job.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,43 +342,76 @@ private string GetParsedModifiedLogEntry(int lineNumber, string rawLogEntry, str
var stringBuilder = new StringBuilder();

// timestamp
stringBuilder.Append(ToUnixTimeStamp(parsedEntry.EdgeServerTimeDelivered) + spaceCharacter);
stringBuilder
.Append(ToUnixTimeStamp(parsedEntry.EdgeServerTimeDelivered))
.Append(spaceCharacter);
// time-taken
stringBuilder.Append((parsedEntry.EdgeServerTimeTaken.HasValue ? parsedEntry.EdgeServerTimeTaken.Value.ToString() : dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.EdgeServerTimeTaken.HasValue ? parsedEntry.EdgeServerTimeTaken.Value.ToString() : dashCharacter))
.Append(spaceCharacter);

// REMOVE c-ip
stringBuilder.Append(dashCharacter + spaceCharacter);
stringBuilder
.Append(dashCharacter)
.Append(spaceCharacter);

// filesize
stringBuilder.Append((parsedEntry.FileSize.HasValue ? parsedEntry.FileSize.Value.ToString() : dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.FileSize.HasValue ? parsedEntry.FileSize.Value.ToString() : dashCharacter))
.Append(spaceCharacter);
// s-ip
stringBuilder.Append((parsedEntry.EdgeServerIpAddress ?? dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.EdgeServerIpAddress ?? dashCharacter))
.Append(spaceCharacter);
// s-port
stringBuilder.Append((parsedEntry.EdgeServerPort.HasValue ? parsedEntry.EdgeServerPort.Value.ToString() : dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.EdgeServerPort.HasValue ? parsedEntry.EdgeServerPort.Value.ToString() : dashCharacter))
.Append(spaceCharacter);
// sc-status
stringBuilder.Append((parsedEntry.CacheStatusCode ?? dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.CacheStatusCode ?? dashCharacter))
.Append(spaceCharacter);
// sc-bytes
stringBuilder.Append((parsedEntry.EdgeServerBytesSent.HasValue ? parsedEntry.EdgeServerBytesSent.Value.ToString() : dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.EdgeServerBytesSent.HasValue ? parsedEntry.EdgeServerBytesSent.Value.ToString() : dashCharacter))
.Append(spaceCharacter);
// cs-method
stringBuilder.Append((parsedEntry.HttpMethod ?? dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.HttpMethod ?? dashCharacter))
.Append(spaceCharacter);
// cs-uri-stem
stringBuilder.Append((parsedEntry.RequestUrl ?? dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.RequestUrl ?? dashCharacter))
.Append(spaceCharacter);

// -
stringBuilder.Append(dashCharacter + spaceCharacter);
stringBuilder
.Append(dashCharacter)
.Append(spaceCharacter);

// rs-duration
stringBuilder.Append((parsedEntry.RemoteServerTimeTaken.HasValue ? parsedEntry.RemoteServerTimeTaken.Value.ToString() : dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.RemoteServerTimeTaken.HasValue ? parsedEntry.RemoteServerTimeTaken.Value.ToString() : dashCharacter))
.Append(spaceCharacter);
// rs-bytes
stringBuilder.Append((parsedEntry.RemoteServerBytesSent.HasValue ? parsedEntry.RemoteServerBytesSent.Value.ToString() : dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.RemoteServerBytesSent.HasValue ? parsedEntry.RemoteServerBytesSent.Value.ToString() : dashCharacter))
.Append(spaceCharacter);
// c-referrer
stringBuilder.Append((parsedEntry.Referrer ?? dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.Referrer ?? dashCharacter))
.Append(spaceCharacter);
// c-user-agent
stringBuilder.Append((parsedEntry.UserAgent ?? dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.UserAgent ?? dashCharacter))
.Append(spaceCharacter);
// customer-id
stringBuilder.Append((parsedEntry.CustomerId ?? dashCharacter) + spaceCharacter);
stringBuilder
.Append((parsedEntry.CustomerId ?? dashCharacter))
.Append(spaceCharacter);
// x-ec_custom-1
stringBuilder.AppendLine((parsedEntry.CustomField ?? dashCharacter) + spaceCharacter);
stringBuilder
.AppendLine((parsedEntry.CustomField ?? dashCharacter));

return stringBuilder.ToString();
}
Expand Down
43 changes: 37 additions & 6 deletions tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
using Moq;
using Stats.AzureCdnLogs.Common;
using Stats.AzureCdnLogs.Common.Collect;
using Stats.CollectAzureChinaCDNLogs;
using Xunit;
Expand All @@ -11,15 +14,20 @@ namespace Tests.Stats.CollectAzureChinaCDNLogs
{
public class ChinaCollectorTests
{
public static IEnumerable<object[]> LogData => new object[][] {
new object[]{ "40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 TCP_MISS/200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na" },
new object[]{ "c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip", null },
new object[]{ "66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - \"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\" na na" },
new object[]{ "66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - \"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\" na na" },
new object[]{ "1.2.3.4,4/6/2019 4:00:20 PM +00:00,GET,\"/v3-flatcontainer/microsoft.codeanalysis.common/1.2.2/microsoft.codeanalysis.common.1.2.2.nupkg\",HTTPS,200,2044843,\"NULL\",\"NuGet VS VSIX/4.7.0 (Microsoft Windows NT 10.0.17134.0, VS Enterprise/15.0)\",796,MISS,4.3.2.1", "1554566420 0 1.2.3.4 0 4.3.2.1 0 MISS/200 2044843 GET /v3-flatcontainer/microsoft.codeanalysis.common/1.2.2/microsoft.codeanalysis.common.1.2.2.nupkg - 796 0 NULL \"NuGet VS VSIX/4.7.0 (Microsoft Windows NT 10.0.17134.0,VS Enterprise/15.0)\" na na" },
new object[]{ "127.0.0.1,1/1/2020 1:23:45 PM +00:00,GET,\"/?q=foo(\"bar\", 2)\",HTTP/1.1,123,456,\"http://nuget.test/?\",\"Mozilla/4.0\",1,HIT,127.0.0.2", null },
};

[Theory]
[InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 TCP_MISS/200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na")]
[InlineData("c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip", null)]
[InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon na na")]
[InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon na na")]
[InlineData("127.0.0.1,1/1/2020 1:23:45 PM +00:00,GET,\"/?q=foo(\"bar\", 2)\",HTTP/1.1,123,456,\"http://nuget.test/?\",\"Mozilla/4.0\",1,HIT,127.0.0.2", null)]
[MemberData(nameof(LogData))]
public void TransformRawLogLine(string input, string expectedOutput)
{
var collector = new ChinaStatsCollector(
var collector = new ChinaStatsCollector(
Mock.Of<ILogSource>(),
Mock.Of<ILogDestination>(),
Mock.Of<ILogger<ChinaStatsCollector>>());
Expand All @@ -28,6 +36,29 @@ public void TransformRawLogLine(string input, string expectedOutput)
string output = tranformedinput == null ? null : tranformedinput.ToString();
Assert.Equal(expectedOutput, output);
}

public static IEnumerable<object[]> InputOnlyLogData => LogData.Select(x => new[] { x[0] });

[Theory]
[MemberData(nameof(InputOnlyLogData))]
public void CdnLogEntryParserIntegration(string input)
{
var collector = new ChinaStatsCollector(
Mock.Of<ILogSource>(),
Mock.Of<ILogDestination>(),
Mock.Of<ILogger<ChinaStatsCollector>>());

var tranformedInput = collector.TransformRawLogLine(input);
if (tranformedInput == null)
{
return;
}
string output = tranformedInput.ToString();
const int lineNumber = 1;
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(lineNumber, output, onErrorAction: null);
Assert.Contains(tranformedInput.XEc_Custom_1, logEntry.CustomField);
Assert.Contains(tranformedInput.CUserAgent, logEntry.UserAgent);
}
}
}