Add support for json logging to all loggers (#4523)

The current support is very minimal and only logs the existing log structure in json instead of plain text. In the future further support for being able to log structs and more complex events could be added.

JSON logging is disabled by default.

CHANGELOG.asciidoc

@@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha2...master[Check the HEAD d
 - New cli subcommands interface. {pull}4420[4420]
 - Allow source path matching in `add_docker_metadata` processor. {pull}4495[4495]
 - Add support for analyzers and multifields in fields.yml. {pull}4574[4574]
+- Add support for JSON logging. {pull}4523[4523]
 
 *Filebeat*
 

auditbeat/auditbeat.reference.yml

@@ -733,3 +733,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

filebeat/filebeat.reference.yml

@@ -1115,3 +1115,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

heartbeat/heartbeat.reference.yml

@@ -888,3 +888,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

libbeat/_meta/config.reference.yml

@@ -674,3 +674,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

libbeat/logp/log.go

@@ -1,10 +1,12 @@
 package logp
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"runtime/debug"
+	"strings"
 	"time"
 )
 
@@ -30,6 +32,7 @@ type logger struct {
 	level             Priority
 	selectors         map[string]struct{}
 	debugAllSelectors bool
+	JSON              bool
 
 	logger  *log.Logger
 	syslog  [LOG_DEBUG + 1]*log.Logger
@@ -73,21 +76,50 @@ func parseSelectors(selectors []string) (map[string]struct{}, bool) {
 
 func debugMessage(calldepth int, selector, format string, v ...interface{}) {
 	if _log.level >= LOG_DEBUG && IsDebug(selector) {
-		send(calldepth+1, LOG_DEBUG, "DBG  ", format, v...)
+		send(calldepth+1, LOG_DEBUG, "DBG", format, v...)
 	}
 }
 
 func send(calldepth int, level Priority, prefix string, format string, v ...interface{}) {
+
+	message := fmt.Sprintf(format, v...)
+	timestamp := time.Now().Format(time.RFC3339)
+	var bytes []byte
+	if _log.JSON {
+		log := map[string]interface{}{
+			"timestamp": timestamp,
+			"level":     prefix,
+			"message":   message,
+		}
+		bytes, _ = json.Marshal(log)
+	} else {
+		// Creates the log message and formats it
+		bytes = []byte(fmt.Sprintf("%s %s %s", timestamp, prefix, message))
+	}
+
 	if _log.toSyslog {
-		_log.syslog[level].Output(calldepth, fmt.Sprintf(format, v...))
+		if _log.JSON {
+			_log.syslog[level].Output(calldepth, string(bytes))
+		} else {
+			_log.syslog[level].Output(calldepth, string(message))
+		}
 	}
 	if _log.toStderr {
-		_log.logger.Output(calldepth, fmt.Sprintf(prefix+format, v...))
+		if _log.JSON {
+			_log.logger.Output(calldepth, string(bytes))
+		} else {
+			_log.logger.Output(calldepth, fmt.Sprintf("%s %s", prefix, message))
+		}
 	}
 	if _log.toFile {
-		// Creates a timestamp for the file log message and formats it
-		prefix = time.Now().Format(time.RFC3339) + " " + prefix
-		_log.rotator.WriteLine([]byte(fmt.Sprintf(prefix+format, v...)))
+		if _log.JSON {
+			_log.rotator.WriteLine(bytes)
+		} else {
+			// Makes sure all prefixes have the same length
+			prefix = prefix + strings.Repeat(" ", 4-len(prefix))
+			bytes = []byte(fmt.Sprintf("%s %s %s", timestamp, prefix, message))
+			_log.rotator.WriteLine(bytes)
+		}
 	}
 }
 
@@ -117,19 +149,19 @@ func msg(level Priority, prefix string, format string, v ...interface{}) {
 }
 
 func Info(format string, v ...interface{}) {
-	msg(LOG_INFO, "INFO ", format, v...)
+	msg(LOG_INFO, "INFO", format, v...)
 }
 
 func Warn(format string, v ...interface{}) {
-	msg(LOG_WARNING, "WARN ", format, v...)
+	msg(LOG_WARNING, "WARN", format, v...)
 }
 
 func Err(format string, v ...interface{}) {
-	msg(LOG_ERR, "ERR ", format, v...)
+	msg(LOG_ERR, "ERR", format, v...)
 }
 
 func Critical(format string, v ...interface{}) {
-	msg(LOG_CRIT, "CRIT ", format, v...)
+	msg(LOG_CRIT, "CRIT", format, v...)
 }
 
 // Deprecate logs a deprecation message.
@@ -152,7 +184,7 @@ func Beta(format string, v ...interface{}) {
 // WTF prints the message at CRIT level and panics immediately with the same
 // message
 func WTF(format string, v ...interface{}) {
-	msg(LOG_CRIT, "CRIT ", format, v)
+	msg(LOG_CRIT, "CRIT", format, v)
 	panic(fmt.Sprintf(format, v...))
 }
 

libbeat/logp/logp.go

@@ -27,6 +27,7 @@ type Logging struct {
 	Files     *FileRotator
 	ToSyslog  *bool `config:"to_syslog"`
 	ToFiles   *bool `config:"to_files"`
+	JSON      bool  `config:"json"`
 	Level     string
 	Metrics   LoggingMetricsConfig `config:"metrics"`
 }
@@ -47,7 +48,6 @@ func init() {
 	verbose = flag.Bool("v", false, "Log at INFO level")
 	toStderr = flag.Bool("e", false, "Log to stderr and disable syslog/file output")
 	debugSelectorsStr = flag.String("d", "", "Enable certain debug selectors")
-
 }
 
 func HandleFlags(name string) error {
@@ -80,7 +80,9 @@ func HandleFlags(name string) error {
 // line flag with a later SetStderr call.
 func Init(name string, config *Logging) error {
 	// reset settings from HandleFlags
-	_log = logger{}
+	_log = logger{
+		JSON: config.JSON,
+	}
 
 	logLevel, err := getLogLevel(config)
 	if err != nil {

metricbeat/metricbeat.reference.yml

@@ -1112,3 +1112,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

packetbeat/packetbeat.reference.yml

@@ -1129,3 +1129,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false

winlogbeat/winlogbeat.reference.yml

@@ -703,3 +703,5 @@ logging.files:
   # Number of rotated log files to keep. Oldest files will be deleted first.
   #keepfiles: 7
 
+# Set to true to log messages in json format.
+#logging.json: false