Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Roll buffer files on size/limit total disk usage in durable mode #92

Merged
merged 9 commits into from
Oct 19, 2017
6 changes: 5 additions & 1 deletion Build.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@ $suffix = @{ $true = ""; $false = "$branch-$revision"}[$branch -eq "master" -and
foreach ($src in ls src/Serilog.*) {
Push-Location $src

& dotnet pack -c Release -o ..\..\.\artifacts --version-suffix=$suffix --include-source
if ($suffix) {
& dotnet pack -c Release -o ..\..\.\artifacts --version-suffix=$suffix --include-source
} else {
& dotnet pack -c Release -o ..\..\.\artifacts --include-source
}
if($LASTEXITCODE -ne 0) { exit 1 }

Pop-Location
Expand Down
6 changes: 1 addition & 5 deletions appveyor.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
version: '{build}'
skip_tags: true
image: Visual Studio 2015
image: Visual Studio 2017
configuration: Release
install:
- ps: mkdir -Force ".\build\" | Out-Null
- ps: Invoke-WebRequest "https://raw.githubusercontent.com/dotnet/cli/rel/1.0.0/scripts/obtain/dotnet-install.ps1" -OutFile ".\build\installcli.ps1"
- ps: $env:DOTNET_INSTALL_DIR = "$pwd\.dotnetcli"
- ps: '& .\build\installcli.ps1 -InstallDir "$env:DOTNET_INSTALL_DIR" -NoPath -Version 1.0.0'
- ps: $env:Path = "$env:DOTNET_INSTALL_DIR;$env:Path"
build_script:
- ps: ./Build.ps1
test: off
Expand Down
4 changes: 4 additions & 0 deletions sample/Sample/Sample.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,8 @@
<ProjectReference Include="..\..\src\Serilog.Sinks.Seq\Serilog.Sinks.Seq.csproj" />
</ItemGroup>

<ItemGroup>
<Folder Include="logs\" />
</ItemGroup>

</Project>
13 changes: 7 additions & 6 deletions src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ public static class SeqLoggerConfigurationExtensions
/// <param name="period">The time to wait between checking for event batches.</param>
/// <param name="bufferBaseFilename">Path for a set of files that will be used to buffer events until they
/// can be successfully transmitted across the network. Individual files will be created using the
/// pattern <paramref name="bufferBaseFilename"/>-{Date}.json.</param>
/// pattern <paramref name="bufferBaseFilename"/>*.json, which should not clash with any other filenames
/// in the same directory.</param>
/// <param name="apiKey">A Seq <i>API key</i> that authenticates the client to the Seq server.</param>
/// <param name="bufferFileSizeLimitBytes">The maximum size, in bytes, to which the buffer
/// <param name="bufferSizeLimitBytes">The maximum amount of data, in bytes, to which the buffer
/// log file for a specific date will be allowed to grow. By default no limit will be applied.</param>
/// <param name="eventBodyLimitBytes">The maximum size, in bytes, that the JSON representation of
/// an event may take before it is dropped rather than being sent to the Seq server. Specify null for no limit.
Expand Down Expand Up @@ -72,7 +73,7 @@ public static LoggerConfiguration Seq(
TimeSpan? period = null,
string apiKey = null,
string bufferBaseFilename = null,
long? bufferFileSizeLimitBytes = null,
long? bufferSizeLimitBytes = null,
long? eventBodyLimitBytes = 256*1024,
LoggingLevelSwitch controlLevelSwitch = null,
HttpMessageHandler messageHandler = null,
Expand All @@ -82,8 +83,8 @@ public static LoggerConfiguration Seq(
{
if (loggerSinkConfiguration == null) throw new ArgumentNullException(nameof(loggerSinkConfiguration));
if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl));
if (bufferFileSizeLimitBytes.HasValue && bufferFileSizeLimitBytes < 0)
throw new ArgumentOutOfRangeException(nameof(bufferFileSizeLimitBytes), "Negative value provided; file size limit must be non-negative.");
if (bufferSizeLimitBytes.HasValue && bufferSizeLimitBytes < 0)
throw new ArgumentOutOfRangeException(nameof(bufferSizeLimitBytes), "Negative value provided; buffer size limit must be non-negative.");
if (queueSizeLimit < 0)
throw new ArgumentOutOfRangeException(nameof(queueSizeLimit), "Queue size limit must be non-zero.");

Expand Down Expand Up @@ -113,7 +114,7 @@ public static LoggerConfiguration Seq(
apiKey,
batchPostingLimit,
defaultedPeriod,
bufferFileSizeLimitBytes,
bufferSizeLimitBytes,
eventBodyLimitBytes,
controlLevelSwitch,
messageHandler,
Expand Down
5 changes: 2 additions & 3 deletions src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

<PropertyGroup>
<Description>Serilog sink that writes to the Seq log server over HTTP/HTTPS.</Description>
<VersionPrefix>3.4.1</VersionPrefix>
<VersionPrefix>4.0.0</VersionPrefix>
<Authors>Serilog Contributors</Authors>
<Copyright>Copyright © Serilog Contributors 2013-2017</Copyright>
<TargetFrameworks>netstandard1.1;netstandard1.3;net45</TargetFrameworks>
Expand Down Expand Up @@ -44,8 +44,7 @@
</ItemGroup>

<ItemGroup Condition=" '$(TargetFramework)' != 'netstandard1.1' ">
<PackageReference Include="Serilog.Sinks.File" Version="3.2.0" />
<PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
<PackageReference Include="Serilog.Sinks.File" Version="4.0.0-dev-00788" />
</ItemGroup>

</Project>
15 changes: 10 additions & 5 deletions src/Serilog.Sinks.Seq/Sinks/Seq/Durable/BookmarkFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,14 @@ public FileSetPosition TryReadBookmark()
{
if (_bookmark.Length != 0)
{
_bookmark.Position = 0;

// Important not to dispose this StreamReader as the stream must remain open.
var reader = new StreamReader(_bookmark, Encoding.UTF8, false, 128);
var current = reader.ReadLine();

if (current != null)
{
_bookmark.Position = 0;
var parts = current.Split(new[] { ":::" }, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 2)
{
Expand All @@ -56,10 +57,14 @@ public void WriteBookmark(FileSetPosition bookmark)
if (bookmark.File == null)
return;

using (var writer = new StreamWriter(_bookmark))
{
writer.WriteLine("{0}:::{1}", bookmark.NextLineStart, bookmark.File);
}
// Don't need to truncate, since we only ever read a single line and
// writes are always newline-terminated
_bookmark.Position = 0;

// Cannot dispose, as `leaveOpen` is not available on all target platforms
var writer = new StreamWriter(_bookmark);
writer.WriteLine("{0}:::{1}", bookmark.NextLineStart, bookmark.File);
writer.Flush();
}

public void Dispose()
Expand Down
30 changes: 18 additions & 12 deletions src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
using System;
using Serilog.Core;
using Serilog.Events;
using Serilog.Sinks.RollingFile;
using System.Net.Http;
using System.Text;

Expand All @@ -26,15 +25,15 @@ namespace Serilog.Sinks.Seq.Durable
class DurableSeqSink : ILogEventSink, IDisposable
{
readonly HttpLogShipper _shipper;
readonly RollingFileSink _sink;
readonly Logger _sink;

public DurableSeqSink(
string serverUrl,
string bufferBaseFilename,
string apiKey,
int batchPostingLimit,
TimeSpan period,
long? bufferFileSizeLimitBytes,
long? bufferSizeLimitBytes,
long? eventBodyLimitBytes,
LoggingLevelSwitch levelControlSwitch,
HttpMessageHandler messageHandler,
Expand All @@ -43,23 +42,30 @@ public DurableSeqSink(
if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl));
if (bufferBaseFilename == null) throw new ArgumentNullException(nameof(bufferBaseFilename));


_shipper = new HttpLogShipper(
serverUrl,
bufferBaseFilename,
apiKey,
batchPostingLimit,
period,
eventBodyLimitBytes,
eventBodyLimitBytes,
levelControlSwitch,
messageHandler,
retainedInvalidPayloadsLimitBytes);
retainedInvalidPayloadsLimitBytes,
bufferSizeLimitBytes);

_sink = new RollingFileSink(
bufferBaseFilename + "-{Date}.json",
new RawJsonFormatter(),
bufferFileSizeLimitBytes,
null,
encoding: Encoding.UTF8);
const long individualFileSizeLimitBytes = 100L * 1024 * 1024;
_sink = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.File(new RawJsonFormatter(),
bufferBaseFilename + "-.json",
rollingInterval: RollingInterval.Day,
fileSizeLimitBytes: individualFileSizeLimitBytes,
rollOnFileSizeLimit: true,
retainedFileCountLimit: null,
encoding: Encoding.UTF8)
.CreateLogger();
}

public void Dispose()
Expand All @@ -74,7 +80,7 @@ public void Emit(LogEvent logEvent)
// are worth the ambiguity.
if (_shipper.IsIncluded(logEvent))
{
_sink.Emit(logEvent);
_sink.Write(logEvent);
}
}
}
Expand Down
80 changes: 47 additions & 33 deletions src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSet.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using Serilog.Debugging;

namespace Serilog.Sinks.Seq.Durable
Expand All @@ -28,6 +29,7 @@ class FileSet
readonly string _bookmarkFilename;
readonly string _candidateSearchPath;
readonly string _logFolder;
readonly Regex _filenameMatcher;

const string InvalidPayloadFilePrefix = "invalid-";

Expand All @@ -37,75 +39,87 @@ public FileSet(string bufferBaseFilename)

_bookmarkFilename = Path.GetFullPath(bufferBaseFilename + ".bookmark");
_logFolder = Path.GetDirectoryName(_bookmarkFilename);
_candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "*.json";
_candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "-*.json";
_filenameMatcher = new Regex("^" + Regex.Escape(Path.GetFileName(bufferBaseFilename)) + "-(?<date>\\d{8})(?<sequence>_[0-9]{3,}){0,1}\\.json$");
}

public BookmarkFile OpenBookmarkFile()
{
return new BookmarkFile(_bookmarkFilename);
}

public string MakeInvalidPayloadFilename(HttpStatusCode statusCode)
public string[] GetBufferFiles()
{
var invalidPayloadFilename = $"{InvalidPayloadFilePrefix}{statusCode}-{Guid.NewGuid():n}.json";
return Path.Combine(_logFolder, invalidPayloadFilename);
return Directory.GetFiles(_logFolder, _candidateSearchPath)
.Select(n => new KeyValuePair<string, Match>(n, _filenameMatcher.Match(Path.GetFileName(n))))
.Where(nm => nm.Value.Success)
.OrderBy(nm => nm.Value.Groups["date"].Value, StringComparer.OrdinalIgnoreCase)
.ThenBy(nm => int.Parse("0" + nm.Value.Groups["sequence"].Value.Replace("_", "")))
.Select(nm => nm.Key)
.ToArray();
}

public void CleanUpInvalidPayloadFiles(long maxNumberOfBytesToRetain)
public void CleanUpBufferFiles(long bufferSizeLimitBytes, int alwaysRetainCount)
{
try
{
var candiateFiles = Directory.EnumerateFiles(_logFolder, $"{InvalidPayloadFilePrefix}*.json");
DeleteOldFiles(maxNumberOfBytesToRetain, candiateFiles);
var bufferFiles = GetBufferFiles();
Array.Reverse(bufferFiles);
DeleteExceedingCumulativeSize(bufferFiles.Select(f => new FileInfo(f)), bufferSizeLimitBytes, 2);
}
catch (Exception ex)
{
SelfLog.WriteLine("Exception thrown while trying to clean up invalid payload files: {0}", ex);
SelfLog.WriteLine("Exception thrown while cleaning up buffer files: {0}", ex);
}
}

public string[] GetFiles()
public string MakeInvalidPayloadFilename(HttpStatusCode statusCode)
{
return Directory.GetFiles(_logFolder, _candidateSearchPath)
.OrderBy(n => n)
.ToArray();
var invalidPayloadFilename = $"{InvalidPayloadFilePrefix}{statusCode}-{Guid.NewGuid():n}.json";
return Path.Combine(_logFolder, invalidPayloadFilename);
}

static void DeleteOldFiles(long maxNumberOfBytesToRetain, IEnumerable<string> files)
public void CleanUpInvalidPayloadFiles(long maxNumberOfBytesToRetain)
{
var orderedFileInfos = from candiateFile in files
let candiateFileInfo = new FileInfo(candiateFile)
orderby candiateFileInfo.LastAccessTimeUtc descending
select candiateFileInfo;

var invalidPayloadFilesToDelete = WhereCumulativeSizeGreaterThan(orderedFileInfos, maxNumberOfBytesToRetain);
try
{
var candidateFiles = from file in Directory.EnumerateFiles(_logFolder, $"{InvalidPayloadFilePrefix}*.json")
let candiateFileInfo = new FileInfo(file)
orderby candiateFileInfo.LastWriteTimeUtc descending
select candiateFileInfo;

foreach (var fileToDelete in invalidPayloadFilesToDelete)
DeleteExceedingCumulativeSize(candidateFiles, maxNumberOfBytesToRetain, 0);
}
catch (Exception ex)
{
try
{
fileToDelete.Delete();
}
catch (Exception ex)
{
SelfLog.WriteLine("Exception '{0}' thrown while trying to delete file {1}", ex.Message, fileToDelete.FullName);
}
SelfLog.WriteLine("Exception thrown while cleaning up invalid payload files: {0}", ex);
}
}
static IEnumerable<FileInfo> WhereCumulativeSizeGreaterThan(IEnumerable<FileInfo> files, long maxCumulativeSize)

static void DeleteExceedingCumulativeSize(IEnumerable<FileInfo> files, long maxNumberOfBytesToRetain, int alwaysRetainCount)
{
long cumulative = 0;
var i = 0;
foreach (var file in files)
{
cumulative += file.Length;
if (cumulative > maxCumulativeSize)

if (i++ < alwaysRetainCount)
continue;

if (cumulative <= maxNumberOfBytesToRetain)
continue;

try
{
yield return file;
file.Delete();
}
catch (Exception ex)
{
SelfLog.WriteLine("Exception thrown while trying to delete file {0}: {1}", file.FullName, ex);
}
}
}

}
}

Expand Down
10 changes: 4 additions & 6 deletions src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSetPosition.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,14 @@ namespace Serilog.Sinks.Seq.Durable
{
struct FileSetPosition
{
readonly string _file;
readonly long _nextLineStart;
public string File { get; }

public string File => _file;
public long NextLineStart => _nextLineStart;
public long NextLineStart { get; }

public FileSetPosition(long nextLineStart, string file)
{
_nextLineStart = nextLineStart;
_file = file;
NextLineStart = nextLineStart;
File = file;
}

public static readonly FileSetPosition None = default(FileSetPosition);
Expand Down
Loading