Added Heroku log formatting for JSON log messages

bin/logagent.js

@@ -76,6 +76,7 @@ var moduleAlias = {
   sql: '../lib/plugins/output-filter/sql.js',
   'access-watch': '../lib/plugins/output-filter/access-watch.js',
   'cloudfoundry-format': '../lib/plugins/output-filter/cloudfoundry-format.js',
+  'heroku-format': '../lib/plugins/output-filter/heroku-format.js',
   'hash-fields': '../lib/plugins/output-filter/hash-fields.js',
   'aes-encrypt-fields': '../lib/plugins/output-filter/aes-encrypt-fields.js',
   'ip-truncate-fields': '../lib/plugins/output-filter/ip-truncate-fields.js',

config/examples/heroku-elasticsearch.yml

@@ -0,0 +1,20 @@
+# Global options
+options:
+  # includeOriginalLine: true
+  # debug: true
+
+input:
+  cloudFoundry:
+    module: input-heroku
+    port: 9999
+
+outputFilter:
+  heroku-format:
+    module: heroku-format
+
+output:
+  elasticsearch:
+    module: elasticsearch
+    url: https://logsene-receiver.sematext.com
+    # debug: true
+

config/examples/heroku-stdout-yml.yml

@@ -74,6 +74,11 @@ parser:
                 }
               }
             }
+
+outputFilter:
+  heroku-format:
+    module: heroku-format
+
 output:
   # print parsed logs in YAML format to stdout if supress is set false    
   stdout: yaml # use 'pretty' for pretty json and 'ldjson' for line delimited json (default)

lib/plugins/input/heroku.js

@@ -4,25 +4,6 @@ const extractTokenRegEx = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9
 const tokenFormatRegEx = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
 const TokenBlacklist = require('../../util/token-blacklist.js')
 
-function jsonParse (text) {
-  try {
-    return JSON.parse(text)
-  } catch (err) {
-    return null
-  }
-}
-
-function extractJson (line, source) {
-  var parsed = {}
-  if (/^\[{0,1}\{.*\}]{0,1}$/.test(line)) {
-    parsed = jsonParse(line)
-    if (!parsed) {
-      return null
-    }
-    return parsed
-  }
-}
-
 function InputHeroku (config, eventEmitter) {
   this.config = config
   this.eventEmitter = eventEmitter
@@ -79,44 +60,6 @@ InputHeroku.prototype.getHttpServer = function (aport, handler) {
   }
 }
 
-function filterHerokuMessage (data, context) {
-  if (data) {
-    data._type = context.sourceName.replace('_' + context.index, '')
-    data.logSource = ('' + data.logSource).replace('_' + context.index, '')
-    var msg = {
-      message: data.message,
-      app: data.app,
-      host: data.host,
-      process_type: data.process_type,
-      originalLine: data.originalLine,
-      severity: data.severity,
-      facility: data.facility
-    }
-    msg.json = extractJson(msg.message)
-    var optionalFields = [
-      'method',
-      'path',
-      'host',
-      'request_id',
-      'fwd',
-      'dyno',
-      'connect',
-      'service',
-      'status',
-      'bytes'
-    ]
-    optionalFields.forEach(function (f) {
-      if (data[f]) {
-        msg[f] = data[f]
-      }
-    })
-    if (!data['@timestamp']) {
-      msg['@timestamp'] = new Date()
-    }
-    return msg
-  }
-}
-
 InputHeroku.prototype.herokuHandler = function (req, res) {
   try {
     var self = this
@@ -166,8 +109,7 @@ InputHeroku.prototype.herokuHandler = function (req, res) {
         }
         self.eventEmitter.emit('data.raw', line, {
           sourceName: 'heroku_' + token,
-          index: token,
-          filter: filterHerokuMessage
+          index: token
         })
       })
       res.end('ok\n')

lib/plugins/output-filter/cloudfoundry-format.js

@@ -32,7 +32,6 @@ function filterCloudFoundryMessage (data, context) {
 }
 
 function parseCloudFoundryTags (tags) {
-  console.log(tags)
   try {
     const parsedTags2dArr = tags
       .split(' ')
@@ -53,7 +52,7 @@ function parseCloudFoundryTags (tags) {
 
     return parsedTags
   } catch (error) {
-    console.log(error)
+    console.error(error)
     return tags
   }
 }

lib/plugins/output-filter/heroku-format.js

@@ -0,0 +1,76 @@
+function jsonParse (text) {
+  try {
+    return JSON.parse(text)
+  } catch (err) {
+    return null
+  }
+}
+
+function extractJson (line, source) {
+  let parsed = {}
+  if (/^\[{0,1}\{.*\}]{0,1}$/.test(line)) {
+    parsed = jsonParse(line)
+    if (!parsed) {
+      return null
+    }
+    return parsed
+  }
+}
+
+function filterHerokuMessage (data, context) {
+  if (data) {
+    data._type = context.sourceName.replace('_' + context.index, '')
+    data.logSource = ('' + data.logSource).replace('_' + context.index, '')
+    const msg = {
+      message: data.message,
+      app: data.app,
+      host: data.host,
+      process_type: data.process_type,
+      originalLine: data.originalLine,
+      severity: data.severity,
+      facility: data.facility
+    }
+
+    const optionalFields = [
+      'method',
+      'path',
+      'host',
+      'request_id',
+      'fwd',
+      'dyno',
+      'connect',
+      'service',
+      'status',
+      'bytes'
+    ]
+    optionalFields.forEach(function (f) {
+      if (data[f]) {
+        msg[f] = data[f]
+      }
+    })
+    if (!data['@timestamp']) {
+      msg['@timestamp'] = new Date()
+    }
+
+    const json = extractJson(msg.message)
+    if (json && json.message) {
+      delete msg.message
+    }
+
+    return {
+      ...msg,
+      ...json
+    }
+  }
+}
+
+module.exports = function (context, config, eventEmitter, log, callback) {
+  try {
+    const parsedLog = filterHerokuMessage(log, context)
+
+    return callback(null, parsedLog)
+  } catch (err) {
+    console.log(err)
+    return callback(null, log)
+  }
+}