diff --git a/global.json b/global.json index 9bf3dd7..6095bd5 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "5.0.202", - "rollForward": "latestPatch" + "version": "5.0.401", + "rollForward": "latestFeature" } } diff --git a/sample/Sample/Program.cs b/sample/Sample/Program.cs index b6fe03d..0fbc07b 100644 --- a/sample/Sample/Program.cs +++ b/sample/Sample/Program.cs @@ -5,7 +5,7 @@ namespace Sample { - public class Program + public static class Program { public static void Main() { diff --git a/sample/Sample/Sample.csproj b/sample/Sample/Sample.csproj index 335c3d0..5c2c58b 100644 --- a/sample/Sample/Sample.csproj +++ b/sample/Sample/Sample.csproj @@ -3,7 +3,7 @@ Sample Console Application nblumhardt - netcoreapp3.1;net47 + net4.8;net5.0 Sample Exe Sample @@ -20,7 +20,7 @@ - + @@ -28,8 +28,4 @@ - - - - diff --git a/serilog-sinks-seq.sln.DotSettings b/serilog-sinks-seq.sln.DotSettings index 6a9f366..dfef825 100644 --- a/serilog-sinks-seq.sln.DotSettings +++ b/serilog-sinks-seq.sln.DotSettings @@ -1,4 +1,5 @@  + True True True True diff --git a/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj b/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj index c4715ca..9f41e00 100644 --- a/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj +++ b/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj @@ -5,7 +5,7 @@ 5.0.2 Serilog Contributors Copyright © Serilog Contributors - netstandard1.1;netstandard1.3;net45;netstandard2.0;netcoreapp3.1;net5.0 + netstandard1.1;netstandard1.3;netstandard2.0;net4.5;netcoreapp3.1;net5.0 true true Serilog @@ -38,11 +38,11 @@ $(DefineConstants);DURABLE;THREADING_TIMER - + $(DefineConstants);DURABLE;THREADING_TIMER;HRESULTS - + diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs index d61219e..83f914e 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs @@ -59,7 +59,6 @@ async Task EmitAsync(LogEvent logEvent) var payload = new StringWriter(); CompactJsonFormatter.FormatEvent(logEvent, payload, JsonValueFormatter); - payload.WriteLine(); var content = new StringContent(payload.ToString(), Encoding.UTF8, SeqApi.CompactLogEventFormatMimeType); if (!string.IsNullOrWhiteSpace(_apiKey)) diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs new file mode 100644 index 0000000..ce1ec09 --- /dev/null +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs @@ -0,0 +1,125 @@ +// Serilog.Sinks.Seq Copyright 2014-2019 Serilog Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Text; +using Serilog.Debugging; +using Serilog.Events; +using Serilog.Formatting; +using Serilog.Formatting.Compact; +using Serilog.Formatting.Json; +using Serilog.Parsing; + +namespace Serilog.Sinks.Seq +{ + /// + /// Wraps a to suppress formatting errors and apply the event body size + /// limit, if any. Placeholder events are logged when an event is unable to be written itself. + /// + class ConstrainedBufferedFormatter : ITextFormatter + { + static readonly int NewLineByteCount = Encoding.UTF8.GetByteCount(Environment.NewLine); + + readonly long? _eventBodyLimitBytes; + readonly CompactJsonFormatter _jsonFormatter = new CompactJsonFormatter(new JsonValueFormatter("$type")); + + public ConstrainedBufferedFormatter(long? eventBodyLimitBytes) + { + _eventBodyLimitBytes = eventBodyLimitBytes; + } + + public void Format(LogEvent logEvent, TextWriter output) + { + Format(logEvent, output, writePlaceholders: true); + } + + void Format(LogEvent logEvent, TextWriter output, bool writePlaceholders) + { + var buffer = new StringWriter(); + + try + { + _jsonFormatter.Format(logEvent, buffer); + } + catch (Exception ex) when (writePlaceholders) + { + SelfLog.WriteLine( + "Event with message template {0} at {1} could not be formatted as JSON and will be dropped: {2}", + logEvent.MessageTemplate.Text, logEvent.Timestamp, ex); + + var placeholder = CreateNonFormattableEventPlaceholder(logEvent, ex); + Format(placeholder, output, writePlaceholders: false); + return; + } + + var jsonLine = buffer.ToString(); + if (CheckEventBodySize(jsonLine, _eventBodyLimitBytes)) + { + output.Write(jsonLine); + } + else + { + SelfLog.WriteLine( + "Event JSON representation exceeds the byte size limit of {0} set for this Seq sink and will be dropped; data: {1}", + _eventBodyLimitBytes, jsonLine); + + if (writePlaceholders) + { + var placeholder = CreateOversizeEventPlaceholder(logEvent, jsonLine, _eventBodyLimitBytes!.Value); + Format(placeholder, output, writePlaceholders: false); + } + } + } + + static LogEvent CreateNonFormattableEventPlaceholder(LogEvent logEvent, Exception ex) + { + return new LogEvent( + logEvent.Timestamp, + LogEventLevel.Error, + ex, + new MessageTemplateParser().Parse("Event with message template {OriginalMessageTemplate} could not be formatted as JSON"), + new[] + { + new LogEventProperty("OriginalMessageTemplate", new ScalarValue(logEvent.MessageTemplate.Text)), + }); + } + + static bool CheckEventBodySize(string jsonLine, long? eventBodyLimitBytes) + { + if (eventBodyLimitBytes == null) + return true; + + var byteCount = Encoding.UTF8.GetByteCount(jsonLine) - NewLineByteCount; + return byteCount <= eventBodyLimitBytes; + } + + static LogEvent CreateOversizeEventPlaceholder(LogEvent logEvent, string jsonLine, long eventBodyLimitBytes) + { + // If the limit is so constrained as to disallow sending 512 bytes + packaging, that's okay - we'll just drop + // the placeholder, too. + var sample = jsonLine.Substring(0, Math.Min(jsonLine.Length, 512)); + return new LogEvent( + logEvent.Timestamp, + LogEventLevel.Error, + exception: null, + new MessageTemplateParser().Parse("Event JSON representation exceeds the body size limit {EventBodyLimitBytes}; sample: {EventBodySample}"), + new[] + { + new LogEventProperty("EventBodyLimitBytes", new ScalarValue(eventBodyLimitBytes)), + new LogEventProperty("EventBodySample", new ScalarValue(sample)), + }); + } + } +} diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs index 8dee47f..e211ee3 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs @@ -19,7 +19,6 @@ using Serilog.Events; using System.Net.Http; using System.Text; -using Serilog.Formatting.Compact; namespace Serilog.Sinks.Seq.Durable { @@ -60,7 +59,7 @@ public DurableSeqSink( const long individualFileSizeLimitBytes = 100L * 1024 * 1024; _sink = new LoggerConfiguration() .MinimumLevel.Verbose() - .WriteTo.File(new CompactJsonFormatter(), + .WriteTo.File(new ConstrainedBufferedFormatter(eventBodyLimitBytes), fileSet.RollingFilePathFormat, rollingInterval: RollingInterval.Day, fileSizeLimitBytes: individualFileSizeLimitBytes, diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqPayloadFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqPayloadFormatter.cs deleted file mode 100644 index 15248fe..0000000 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqPayloadFormatter.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Serilog.Sinks.Seq Copyright 2014-2019 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using Serilog.Debugging; -using Serilog.Events; -using Serilog.Formatting.Compact; -using Serilog.Formatting.Json; - -namespace Serilog.Sinks.Seq -{ - static class SeqPayloadFormatter - { - static readonly JsonValueFormatter JsonValueFormatter = new JsonValueFormatter("$type"); - - public static string FormatCompactPayload(IEnumerable events, long? eventBodyLimitBytes) - { - var payload = new StringWriter(); - - foreach (var logEvent in events) - { - var buffer = new StringWriter(); - - try - { - CompactJsonFormatter.FormatEvent(logEvent, buffer, JsonValueFormatter); - } - catch (Exception ex) - { - LogNonFormattableEvent(logEvent, ex); - continue; - } - - var json = buffer.ToString(); - if (CheckEventBodySize(json, eventBodyLimitBytes)) - { - payload.WriteLine(json); - } - } - - return payload.ToString(); - } - - static void LogNonFormattableEvent(LogEvent logEvent, Exception ex) - { - SelfLog.WriteLine( - "Event at {0} with message template {1} could not be formatted into JSON for Seq and will be dropped: {2}", - logEvent.Timestamp.ToString("o"), logEvent.MessageTemplate.Text, ex); - } - - static bool CheckEventBodySize(string json, long? eventBodyLimitBytes) - { - if (eventBodyLimitBytes.HasValue && - Encoding.UTF8.GetByteCount(json) > eventBodyLimitBytes.Value) - { - SelfLog.WriteLine( - "Event JSON representation exceeds the byte size limit of {0} set for this Seq sink and will be dropped; data: {1}", - eventBodyLimitBytes, json); - return false; - } - - return true; - } - - } -} \ No newline at end of file diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs index 9ea9540..6f6a3f3 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs @@ -14,6 +14,7 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net.Http; using System.Text; @@ -33,7 +34,7 @@ class SeqSink : IBatchedLogEventSink, IDisposable static readonly TimeSpan RequiredLevelCheckInterval = TimeSpan.FromMinutes(2); readonly string _apiKey; - readonly long? _eventBodyLimitBytes; + readonly ConstrainedBufferedFormatter _formatter; readonly HttpClient _httpClient; DateTime _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval); @@ -49,9 +50,9 @@ public SeqSink( if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); _controlledSwitch = controlledSwitch ?? throw new ArgumentNullException(nameof(controlledSwitch)); _apiKey = apiKey; - _eventBodyLimitBytes = eventBodyLimitBytes; _httpClient = messageHandler != null ? new HttpClient(messageHandler) : new HttpClient(); _httpClient.BaseAddress = new Uri(SeqApi.NormalizeServerBaseAddress(serverUrl)); + _formatter = new ConstrainedBufferedFormatter(eventBodyLimitBytes); } public void Dispose() @@ -74,10 +75,13 @@ public async Task EmitBatchAsync(IEnumerable events) { _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval); - var payloadContentType = SeqApi.CompactLogEventFormatMimeType; - var payload = SeqPayloadFormatter.FormatCompactPayload(events, _eventBodyLimitBytes); + var payload = new StringWriter(); + foreach (var evt in events) + { + _formatter.Format(evt, payload); + } - var content = new StringContent(payload, Encoding.UTF8, payloadContentType); + var content = new StringContent(payload.ToString(), Encoding.UTF8, SeqApi.CompactLogEventFormatMimeType); if (!string.IsNullOrWhiteSpace(_apiKey)) content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); diff --git a/test/Serilog.Sinks.Seq.Tests/Audit/SeqAuditSinkTests.cs b/test/Serilog.Sinks.Seq.Tests/Audit/SeqAuditSinkTests.cs index 642b5aa..3e354c0 100644 --- a/test/Serilog.Sinks.Seq.Tests/Audit/SeqAuditSinkTests.cs +++ b/test/Serilog.Sinks.Seq.Tests/Audit/SeqAuditSinkTests.cs @@ -20,7 +20,7 @@ public void EarlyCommunicationErrorsPropagateToCallerWhenAuditing() } } - [Fact] + [Fact] // This test requires an outbound connection in order to execute properly. public void RemoteCommunicationErrorsPropagateToCallerWhenAuditing() { using (var logger = new LoggerConfiguration() diff --git a/test/Serilog.Sinks.Seq.Tests/ConstrainedBufferedFormatterTests.cs b/test/Serilog.Sinks.Seq.Tests/ConstrainedBufferedFormatterTests.cs new file mode 100644 index 0000000..9b1b0e3 --- /dev/null +++ b/test/Serilog.Sinks.Seq.Tests/ConstrainedBufferedFormatterTests.cs @@ -0,0 +1,44 @@ +using System.IO; +using Serilog.Sinks.Seq.Tests.Support; +using Xunit; + +namespace Serilog.Sinks.Seq.Tests +{ + public class ConstrainedBufferedFormatterTests + { + [Fact] + public void EventsAreFormattedIntoCompactJsonPayloads() + { + var evt = Some.LogEvent("Hello, {Name}!", "Alice"); + var formatter = new ConstrainedBufferedFormatter(null); + var json = new StringWriter(); + formatter.Format(evt, json); + Assert.Contains("Name\":\"Alice", json.ToString()); + } + + [Fact] + public void PlaceholdersAreLoggedWhenCompactJsonRenderingFails() + { + var evt = Some.LogEvent(new NastyException(), "Hello, {Name}!", "Alice"); + var formatter = new ConstrainedBufferedFormatter(null); + var json = new StringWriter(); + formatter.Format(evt, json); + var jsonString = json.ToString(); + Assert.Contains("could not be formatted", jsonString); + Assert.Contains("OriginalMessageTemplate\":\"Hello, ", jsonString); + } + + [Fact] + public void PlaceholdersAreLoggedWhenTheEventSizeLimitIsExceeded() + { + var evt = Some.LogEvent("Hello, {Name}!", new string('a', 10000)); + var formatter = new ConstrainedBufferedFormatter(2000); + var json = new StringWriter(); + formatter.Format(evt, json); + var jsonString = json.ToString(); + Assert.Contains("exceeds the body size limit", jsonString); + Assert.Contains("\"EventBodySample\"", jsonString); + Assert.Contains("aaaaa", jsonString); + } + } +} diff --git a/test/Serilog.Sinks.Seq.Tests/SeqPayloadFormatterTests.cs b/test/Serilog.Sinks.Seq.Tests/SeqPayloadFormatterTests.cs deleted file mode 100644 index 69fde7d..0000000 --- a/test/Serilog.Sinks.Seq.Tests/SeqPayloadFormatterTests.cs +++ /dev/null @@ -1,24 +0,0 @@ -using Serilog.Sinks.Seq.Tests.Support; -using Xunit; - -namespace Serilog.Sinks.Seq.Tests -{ - public class SeqPayloadFormatterTests - { - [Fact] - public void EventsAreFormattedIntoCompactJsonPayloads() - { - var evt = Some.LogEvent("Hello, {Name}!", "Alice"); - var json = SeqPayloadFormatter.FormatCompactPayload(new[] { evt }, null); - Assert.Contains("Name\":\"Alice", json); - } - - [Fact] - public void EventsAreDroppedWhenCompactJsonRenderingFails() - { - var evt = Some.LogEvent(new NastyException(), "Hello, {Name}!", "Alice"); - var json = SeqPayloadFormatter.FormatCompactPayload(new[] { evt }, null); - Assert.Empty(json); - } - } -}