-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Custom sink for handling large audit events (#15)
* added structure for the additional serilog sink for blob and eventhub * added addition blob storage serilog sink. now audit events would go to both eventhub and blob destinations * Separate extension logger methods created for blob and eventhub for the purpose of conditional logging * Adding custom sink * adding blob upload * upload blob with name as event id * Renaming project * restructured the folders * adding event hub upload * Adding message check * upgrade image to chcek if build is passing * reverting the image upgrade as build failed * configurable event size limit and renaming variables * removing unwanted files * changing variable name * audit client * pr comments * pr comments * pr comments * pr comments * library updated * fixing appveyor build issues * code fix for build failure * pr review comments * removing blob name from class variable * pr comments * renaming blob name * removing unused package reference * remove unwanted interfaces * refactoring * pr comments * renaming variable * added extension method accepting blob client * pr comments * pr comments Co-authored-by: Avin Mathew (UYW) <UYW@kmd.dk> Co-authored-by: Satyajit <YYW@kmd.dk>
- Loading branch information
3 people
authored
Nov 3, 2020
1 parent
3ce7c91
commit 7dff721
Showing
15 changed files
with
656 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
67 changes: 67 additions & 0 deletions
67
src/Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink/AuditEventPayload.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Globalization; | ||
| using System.IO; | ||
| using System.Text; | ||
| using Serilog.Events; | ||
| using Serilog.Formatting; | ||
|
|
||
| namespace Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink | ||
| { | ||
| /// <summary> | ||
| /// This class handles methods related to Audit event payload | ||
| /// </summary> | ||
| public static class AuditEventPayload | ||
| { | ||
| public const string MessageTemplate = "Event details for event id {0} can be found at {1}"; | ||
|
|
||
| /// <summary> | ||
| /// This method checks if audit event payload size is more than the limit or not | ||
| /// </summary> | ||
| /// <param name="textFormatter">Formatter</param> | ||
| /// <param name="logEvent">Log event</param> | ||
| /// <param name="eventSizeLimit">Event size limit</param> | ||
| /// <returns>Returns if message size is greater than the limit</returns> | ||
| public static bool DoesAuditEventPayloadExceedLimit(ITextFormatter textFormatter, LogEvent logEvent, int eventSizeLimit) | ||
| { | ||
| string content; | ||
| using (var render = new StringWriter()) | ||
| { | ||
| textFormatter.Format(logEvent, render); | ||
| content = render.ToString(); | ||
| } | ||
|
|
||
| if (Encoding.UTF8.GetByteCount(content) > eventSizeLimit) | ||
| { | ||
| return true; | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Transform the logevent to provide different message with blob url property added | ||
| /// </summary> | ||
| /// <param name="logEvent">Log event</param> | ||
| /// <param name="blobUrl">Blob url</param> | ||
| /// <returns>New log event after message transformation</returns> | ||
| public static LogEvent AuditEventMessageTransformation(LogEvent logEvent, Uri blobUrl) | ||
| { | ||
| var properties = new List<LogEventProperty>(); | ||
| foreach (var property in logEvent.Properties) | ||
| { | ||
| var logEventProperty = new LogEventProperty(property.Key, property.Value); | ||
| properties.Add(logEventProperty); | ||
| } | ||
|
|
||
| var newLogEvent = new LogEvent( | ||
| logEvent.Timestamp, | ||
| logEvent.Level, | ||
| logEvent.Exception, | ||
| new MessageTemplate(string.Format(CultureInfo.InvariantCulture, MessageTemplate, logEvent.Properties["_EventId"], blobUrl), logEvent.MessageTemplate.Tokens), | ||
| properties); | ||
|
|
||
| return newLogEvent; | ||
| } | ||
| } | ||
| } |
64 changes: 64 additions & 0 deletions
64
src/Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink/AzureBlobOrEventHubCustomSink.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| using System; | ||
| using Azure.Storage.Blobs; | ||
| using Microsoft.Azure.EventHubs; | ||
| using Serilog.Core; | ||
| using Serilog.Events; | ||
| using Serilog.Formatting; | ||
| using Serilog.Sinks.AzureEventHub; | ||
|
|
||
| namespace Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink | ||
| { | ||
| public class AzureBlobOrEventHubCustomSink : ILogEventSink | ||
| { | ||
| private readonly BlobServiceClient blobServiceClient; | ||
| private readonly ITextFormatter textFormatter; | ||
| private readonly int eventSizeLimit; | ||
| private readonly string storageContainerName; | ||
| private readonly AzureEventHubSink azureEventHubSink; | ||
|
|
||
| public AzureBlobOrEventHubCustomSink( | ||
| BlobServiceClient blobServiceClient, | ||
| EventHubClient eventHubClient, | ||
| ITextFormatter textFormatter, | ||
| int eventSizeLimit = 256 * 1024, | ||
| string storageContainerName = "logs") | ||
| { | ||
| this.textFormatter = textFormatter; | ||
| this.blobServiceClient = blobServiceClient; | ||
| this.storageContainerName = storageContainerName; | ||
| this.eventSizeLimit = eventSizeLimit; | ||
| this.azureEventHubSink = new AzureEventHubSink(eventHubClient, textFormatter); | ||
| } | ||
|
|
||
| public void Emit(LogEvent logEvent) | ||
| { | ||
| if (AuditEventPayload.DoesAuditEventPayloadExceedLimit(this.textFormatter, logEvent, this.eventSizeLimit)) | ||
| { | ||
| var blobUrl = this.UploadToBlob(logEvent); | ||
| logEvent = AuditEventPayload.AuditEventMessageTransformation(logEvent, blobUrl); | ||
| } | ||
|
|
||
| this.azureEventHubSink.Emit(logEvent); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Handles uploading to Azure blob | ||
| /// </summary> | ||
| /// <param name="logEvent">Log event</param> | ||
| /// <returns>blob url</returns> | ||
| private Uri UploadToBlob(LogEvent logEvent) | ||
| { | ||
| var content = AzureBlobServiceHelper.PrepareBlobContentForUpload(this.textFormatter, logEvent); | ||
| var eventId = string.Empty; | ||
|
|
||
| // Get Event Id value and use it as blob name | ||
| logEvent.Properties.TryGetValue("_EventId", out var eventIdValue); | ||
| if (eventIdValue != null) | ||
| { | ||
| eventId = eventIdValue.ToString(); | ||
| } | ||
|
|
||
| return AzureBlobServiceProvider.UploadBlob(this.blobServiceClient, this.storageContainerName, eventId, content); | ||
| } | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
src/Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink/AzureBlobServiceHelper.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| using System; | ||
| using System.IO; | ||
| using Serilog.Events; | ||
| using Serilog.Formatting; | ||
|
|
||
| namespace Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink | ||
| { | ||
| public static class AzureBlobServiceHelper | ||
| { | ||
| public static string PrepareBlobContentForUpload(ITextFormatter textFormatter, LogEvent logEvent) | ||
| { | ||
| if (textFormatter == null) | ||
| { | ||
| throw new ArgumentNullException(nameof(textFormatter)); | ||
| } | ||
|
|
||
| if (logEvent == null) | ||
| { | ||
| throw new ArgumentNullException(nameof(logEvent)); | ||
| } | ||
|
|
||
| string content; | ||
|
|
||
| using (var tempStringWriter = new StringWriter()) | ||
| { | ||
| try | ||
| { | ||
| textFormatter.Format(logEvent, tempStringWriter); | ||
| tempStringWriter.Flush(); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| Serilog.Debugging.SelfLog.WriteLine($"Exception {ex} thrown during logEvent formatting. The log event will be dropped."); | ||
| } | ||
|
|
||
| content = tempStringWriter.ToString(); | ||
| } | ||
|
|
||
| return content; | ||
| } | ||
| } | ||
| } |
58 changes: 58 additions & 0 deletions
58
src/Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink/AzureBlobServiceProvider.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| using System; | ||
| using System.IO; | ||
| using System.Text; | ||
| using Azure.Storage.Blobs; | ||
|
|
||
| namespace Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink | ||
| { | ||
| public static class AzureBlobServiceProvider | ||
| { | ||
| public const string PathDivider = "/"; | ||
|
|
||
| /// <summary> | ||
| /// Uploads blob and return the blob url | ||
| /// </summary> | ||
| /// <param name="blobServiceClient">Blob service client</param> | ||
| /// <param name="blobContainerName">Container name</param> | ||
| /// <param name="eventId">Event id from log context</param> | ||
| /// <param name="content">Content to be uploaded</param> | ||
| /// <returns>Blob url</returns> | ||
| public static Uri UploadBlob(BlobServiceClient blobServiceClient, string blobContainerName, string eventId, string content) | ||
| { | ||
| // Get a reference to a blob | ||
| var blobClient = GetBlobClient(blobServiceClient, blobContainerName, eventId); | ||
| using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) | ||
| { | ||
| try | ||
| { | ||
| blobClient.Upload(stream); | ||
| return blobClient.Uri; | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| Serilog.Debugging.SelfLog.WriteLine($"Exception {ex} thrown while trying to upload blob."); | ||
| return null; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Check if container or blob exists or not and return a blob client object accordingly | ||
| /// </summary> | ||
| /// <param name="blobServiceClient">Blob service client</param> | ||
| /// <param name="blobContainerName">Container name</param> | ||
| /// <param name="eventId">Event id from log context</param> | ||
| /// <returns>Blob client</returns> | ||
| private static BlobClient GetBlobClient(BlobServiceClient blobServiceClient, string blobContainerName, string eventId) | ||
| { | ||
| var dt = DateTimeOffset.UtcNow; | ||
| var containerClient = blobServiceClient.GetBlobContainerClient(blobContainerName); | ||
| containerClient.CreateIfNotExists(); | ||
|
|
||
| // If event id is empty then create a new guid. This scenario is remotely likely to happen | ||
| eventId = eventId ?? Guid.NewGuid().ToString(); | ||
| eventId = $"{dt:yyyy}{PathDivider}{dt:MM}{PathDivider}{dt:dd}{PathDivider}{dt:yyyyMMdd_HHMM}_{eventId}.log"; | ||
| return containerClient.GetBlobClient(eventId); | ||
| } | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
...udit.Client.AzureBlobOrEventHubSink/Kmd.Logic.Audit.Client.AzureBlobOrEventHubSink.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Library</OutputType> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Azure.Storage.Blobs" Version="12.6.0" /> | ||
| <PackageReference Include="Microsoft.Azure.EventHubs" Version="4.3.0" /> | ||
| <PackageReference Include="Serilog" Version="2.10.0" /> | ||
| <PackageReference Include="Serilog.Sinks.AzureEventHub" Version="5.0.0" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
Oops, something went wrong.