Save logevent as JSON

README.md

@@ -7,15 +7,38 @@ A Serilog sink that writes events to Microsoft SQL Server. While a NoSql store a
 **Package** - [Serilog.Sinks.MSSqlServer](http://nuget.org/packages/serilog.sinks.mssqlserver)
 | **Platforms** - .NET 4.5
 
-You'll need to create a database and add a table like the one you can find in this [Gist](https://gist.github.com/mivano/10429656). 
-
 ```csharp
 var log = new LoggerConfiguration()
     .WriteTo.MSSqlServer(connectionString: @"Server=...", tableName: "Logs")
     .CreateLogger();
 ```
 
-Make sure to set up security in such a way that the sink can write to the log table. If you don't plan on using the properties, then you can disable the storage of them. 
+You'll need to create a table like this in your database:
+
+```
+CREATE TABLE [Logs] (
+
+   [Id] int IDENTITY(1,1) NOT NULL,
+   [Message] nvarchar(max) NULL,
+   [MessageTemplate] nvarchar(max) NULL,
+   [Level] nvarchar(128) NULL,
+   [TimeStamp] datetimeoffset(7) NOT NULL,  -- use datetime for SQL Server pre-2008
+   [Exception] nvarchar(max) NULL,
+   [Properties] xml NULL,
+   [LogEvent] nvarchar(max) NULL
+
+   CONSTRAINT [PK_Logs] 
+     PRIMARY KEY CLUSTERED ([Id] ASC) 
+	 WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) 
+     ON [PRIMARY]
+
+) ON [PRIMARY];
+```
+
+Make sure to set up security in such a way that the sink can write to the log table. 
+
+If you don't plan on using the Properties or LogEvent columns, you can disable their use with the *storeProperties* and *storeLogEvent* parameters.
+
 
 ### XML configuration
 
@@ -43,3 +66,16 @@ var log = new LoggerConfiguration()
     .CreateLogger();
 ```
 The log event properties `User` and `Other` will now be placed in the corresponding column upon logging. The property name must match a column name in your table.
+
+
+#### Excluding redundant items from the Properties column
+
+By default the additional properties will still be included in the XML data saved to the Properties column (assuming that is not disabled via the storeProperties parameter). That's consistent with the idea behind structured logging, and makes it easier to convert the log data to another (e.g. NoSql) storage platform later if desired.  
+
+However, if the data is to stay in SQL Server, then the additional properties may not need to be saved in both columns and XML.  Use the *excludeAdditionalProperties* parameter in the sink configuration to exclude the redundant properties from the XML.
+
+
+### Saving the Log Event data
+
+By default the log event JSON is stored to the LogEvent column.  This can be disabled with the *storeLogEvent* parameter.
+

src/Serilog.Sinks.MSSqlServer/LoggerConfigurationMSSqlServerExtensions.cs

@@ -40,24 +40,42 @@ public static class LoggerConfigurationMSSqlServerExtensions
         /// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
         /// <param name="storeTimestampInUtc">Store Timestamp In UTC</param>
         /// <param name="additionalDataColumns">Additional columns for data storage.</param>
+        /// <param name="excludeAdditionalProperties">Exclude properties from the Properties column if they are being saved to additional columns.</param>
+        /// <param name="storeLogEvent">Save the entire log event to the LogEvent column (nvarchar) as JSON.</param>
         /// <returns>Logger configuration, allowing configuration to continue.</returns>
         /// <exception cref="ArgumentNullException">A required parameter is null.</exception>
         public static LoggerConfiguration MSSqlServer(
             this LoggerSinkConfiguration loggerConfiguration,
-            string connectionString, string tableName, bool storeProperties = true,
+            string connectionString,
+            string tableName,
+            bool storeProperties = true,
             LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
             int batchPostingLimit = MSSqlServerSink.DefaultBatchPostingLimit,
             TimeSpan? period = null,
             IFormatProvider formatProvider = null,
             bool storeTimestampInUtc = false,
-            DataColumn[] additionalDataColumns = null)
+            DataColumn[] additionalDataColumns = null,
+            bool excludeAdditionalProperties = false,
+            bool storeLogEvent = true
+            )
         {
             if (loggerConfiguration == null) throw new ArgumentNullException("loggerConfiguration");
 
             var defaultedPeriod = period ?? MSSqlServerSink.DefaultPeriod;
 
             return loggerConfiguration.Sink(
-                new MSSqlServerSink(connectionString, tableName, storeProperties, batchPostingLimit, defaultedPeriod, formatProvider, storeTimestampInUtc, additionalDataColumns),
+                new MSSqlServerSink(
+                    connectionString,
+                    tableName,
+                    storeProperties,
+                    batchPostingLimit,
+                    defaultedPeriod,
+                    formatProvider,
+                    storeTimestampInUtc,
+                    additionalDataColumns,
+                    excludeAdditionalProperties,
+                    storeLogEvent
+                    ),
                 restrictedToMinimumLevel);
         }
     }

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/MSSqlServerSink.cs

@@ -16,10 +16,12 @@
 using System.Collections.Generic;
 using System.Data;
 using System.Data.SqlClient;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Serilog.Events;
+using Serilog.Formatting.Json;
 using Serilog.Sinks.PeriodicBatching;
 
 namespace Serilog.Sinks.MSSqlServer
@@ -49,7 +51,12 @@ public class MSSqlServerSink : PeriodicBatchingSink
         readonly CancellationTokenSource _token = new CancellationTokenSource();
         readonly bool _storeTimestampInUtc;
 
-        private DataColumn[] _additionalDataColumns;
+        private readonly DataColumn[] _additionalDataColumns;
+        private readonly bool _excludeAdditionalProperties;
+        private readonly HashSet<string> _additionalDataColumnNames;
+
+        private readonly bool _storeLogEvent;
+        private readonly JsonFormatter _jsonFormatter;
 
         /// <summary>
         ///     Construct a sink posting to the specified database.
@@ -62,8 +69,20 @@ public class MSSqlServerSink : PeriodicBatchingSink
         /// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
         /// <param name="storeTimestampInUtc">Store Timestamp In UTC</param>
         /// <param name="additionalDataColumns">Additional columns for data storage.</param>
-        public MSSqlServerSink(string connectionString, string tableName, bool includeProperties, int batchPostingLimit,
-            TimeSpan period, IFormatProvider formatProvider, bool storeTimestampInUtc, DataColumn[] additionalDataColumns = null )
+        /// <param name="excludeAdditionalProperties">Exclude properties from the Properties column if they are being saved to additional columns.</param>
+        /// <param name="storeLogEvent">Save the entire log event to the LogEvent column (nvarchar) as JSON.</param>
+        public MSSqlServerSink(
+            string connectionString,
+            string tableName,
+            bool includeProperties,
+            int batchPostingLimit,
+            TimeSpan period,
+            IFormatProvider formatProvider,
+            bool storeTimestampInUtc,
+            DataColumn[] additionalDataColumns = null,
+            bool excludeAdditionalProperties = false,
+            bool storeLogEvent = true
+            )
             : base(batchPostingLimit, period)
         {
             if (string.IsNullOrWhiteSpace(connectionString))
@@ -72,13 +91,19 @@ public MSSqlServerSink(string connectionString, string tableName, bool includePr
             if (string.IsNullOrWhiteSpace(tableName))
                 throw new ArgumentNullException("tableName");
 
-
             _connectionString = connectionString;
             _tableName = tableName;
             _includeProperties = includeProperties;
             _formatProvider = formatProvider;
             _storeTimestampInUtc = storeTimestampInUtc;
             _additionalDataColumns = additionalDataColumns;
+            if (_additionalDataColumns != null)
+                _additionalDataColumnNames = new HashSet<string>(_additionalDataColumns.Select(c => c.ColumnName), StringComparer.OrdinalIgnoreCase);
+            _excludeAdditionalProperties = excludeAdditionalProperties;
+
+            _storeLogEvent = storeLogEvent;
+            if (_storeLogEvent)
+                _jsonFormatter = new JsonFormatter(formatProvider: formatProvider);
 
             // Prepare the data table
             _eventsTable = CreateDataTable();
@@ -166,7 +191,14 @@ DataTable CreateDataTable()
             };
             eventsTable.Columns.Add(props);
 
-            if ( _additionalDataColumns != null )
+            var eventData = new DataColumn
+            {
+                DataType = Type.GetType("System.String"),
+                ColumnName = "LogEvent"
+            };
+            eventsTable.Columns.Add(eventData);
+
+            if (_additionalDataColumns != null)
             {
                 eventsTable.Columns.AddRange(_additionalDataColumns);
             }
@@ -179,7 +211,7 @@ DataTable CreateDataTable()
             return eventsTable;
         }
 
-     void FillDataTable(IEnumerable<LogEvent> events)
+        void FillDataTable(IEnumerable<LogEvent> events)
         {
             // Add the new rows to the collection. 
             foreach (var logEvent in events)
@@ -188,28 +220,31 @@ void FillDataTable(IEnumerable<LogEvent> events)
                 row["Message"] = logEvent.RenderMessage(_formatProvider);
                 row["MessageTemplate"] = logEvent.MessageTemplate;
                 row["Level"] = logEvent.Level;
-                row["TimeStamp"] = (_storeTimestampInUtc) ? logEvent.Timestamp.DateTime.ToUniversalTime() 
-                                                          : logEvent.Timestamp.DateTime;
+                row["TimeStamp"] = (_storeTimestampInUtc)
+                    ? logEvent.Timestamp.DateTime.ToUniversalTime()
+                    : logEvent.Timestamp.DateTime;
                 row["Exception"] = logEvent.Exception != null ? logEvent.Exception.ToString() : null;
 
                 if (_includeProperties)
-                {
                     row["Properties"] = ConvertPropertiesToXmlStructure(logEvent.Properties);
-                }
-                if ( _additionalDataColumns != null )
-                {
-                    ConvertPropertiesToColumn( row, logEvent.Properties );
-                }
+
+                if (_storeLogEvent)
+                    row["LogEvent"] = LogEventToJson(logEvent);
+
+                if (_additionalDataColumns != null)
+                    ConvertPropertiesToColumn(row, logEvent.Properties);
 
                 _eventsTable.Rows.Add(row);
             }
 
             _eventsTable.AcceptChanges();
         }
 
-        static string ConvertPropertiesToXmlStructure(
-            IEnumerable<KeyValuePair<string, LogEventPropertyValue>> properties)
+        private string ConvertPropertiesToXmlStructure(IEnumerable<KeyValuePair<string, LogEventPropertyValue>> properties)
         {
+            if (_excludeAdditionalProperties)
+                properties = properties.Where(p => !_additionalDataColumnNames.Contains(p.Key));
+
             var sb = new StringBuilder();
 
             sb.Append("<properties>");
@@ -225,6 +260,14 @@ static string ConvertPropertiesToXmlStructure(
             return sb.ToString();
         }
 
+        private string LogEventToJson(LogEvent logEvent)
+        {
+            var sb = new StringBuilder();
+            using (var writer = new System.IO.StringWriter(sb))
+                _jsonFormatter.Format(logEvent, writer);
+            return sb.ToString();
+        }
+
         /// <summary>
         ///     Mapping values from properties which have a corresponding data row.
         ///     Matching is done based on Column name and property key
@@ -234,13 +277,8 @@ static string ConvertPropertiesToXmlStructure(
         private void ConvertPropertiesToColumn(
             DataRow row, IReadOnlyDictionary<string, LogEventPropertyValue> properties)
         {
-            foreach (var property in properties)
-            {
-                if (row.Table.Columns.Contains(property.Key))
-                {
-                    row[property.Key] = property.Value.ToString();
-                }
-            }
+            foreach (var property in properties.Where(p => _additionalDataColumnNames.Contains(p.Key)))
+                row[property.Key] = property.Value.ToString();
         }
 
         /// <summary>
@@ -252,9 +290,9 @@ protected override void Dispose(bool disposing)
             _token.Cancel();
 
             if (_eventsTable != null)
-                _eventsTable.Dispose();           
+                _eventsTable.Dispose();
 
             base.Dispose(disposing);
         }
     }
-}
+}