initial commit for heroku log format

bin/logagent.js

@@ -76,6 +76,7 @@ var moduleAlias = {
   sql: '../lib/plugins/output-filter/sql.js',
   'access-watch': '../lib/plugins/output-filter/access-watch.js',
   'cloudfoundry-format': '../lib/plugins/output-filter/cloudfoundry-format.js',
+  'heroku-format': '../lib/plugins/output-filter/heroku-format.js',
   'hash-fields': '../lib/plugins/output-filter/hash-fields.js',
   'aes-encrypt-fields': '../lib/plugins/output-filter/aes-encrypt-fields.js',
   'ip-truncate-fields': '../lib/plugins/output-filter/ip-truncate-fields.js',

config/examples/heroku-elasticsearch.yml

@@ -0,0 +1,20 @@
+# Global options
+options:
+  # includeOriginalLine: true
+  # debug: true
+
+input:
+  cloudFoundry:
+    module: input-heroku
+    port: 9999
+
+outputFilter:
+  heroku-format:
+    module: heroku-format
+
+output:
+  elasticsearch:
+    module: elasticsearch
+    url: https://logsene-receiver.sematext.com
+    # debug: true
+

lib/plugins/output-filter/cloudfoundry-format.js

@@ -32,7 +32,6 @@ function filterCloudFoundryMessage (data, context) {
 }
 
 function parseCloudFoundryTags (tags) {
-  console.log(tags)
   try {
     const parsedTags2dArr = tags
       .split(' ')
@@ -53,7 +52,7 @@ function parseCloudFoundryTags (tags) {
 
     return parsedTags
   } catch (error) {
-    console.log(error)
+    console.error(error)
     return tags
   }
 }

lib/plugins/output-filter/heroku-format.js

@@ -0,0 +1,42 @@
+function jsonParse (text) {
+  try {
+    return JSON.parse(text)
+  } catch (err) {
+    return null
+  }
+}
+
+function parseHerokuMessage (data, context) {
+  if (data) {
+    data._type = context.sourceName.replace('_' + context.index, '')
+    data.logSource = ('' + data.logSource).replace('_' + context.index, '')
+    if (!data['@timestamp']) {
+      data['@timestamp'] = new Date()
+    }
+    if (data.message) {
+      data.json = jsonParse(data.message)
+    }
+  }
+  return data
+}
+
+module.exports = function (context, config, eventEmitter, log, callback) {
+  try {
+    const parsedLog = parseHerokuMessage(log, context)
+    const json = parsedLog.json
+    const message = parsedLog.message
+    delete parsedLog.json
+    delete parsedLog.message
+
+    const structuredLog = {
+      message,
+      ...json,
+      ...parsedLog
+    }
+
+    return callback(null, structuredLog)
+  } catch (err) {
+    console.error(err)
+    return callback(null, log)
+  }
+}