From 0422bc08739fbb44493d5dff0c99e4cf1188eaaa Mon Sep 17 00:00:00 2001 From: Adnan Rahic Date: Thu, 26 Nov 2020 18:39:55 +0100 Subject: [PATCH 1/5] initial commit for heroku log format --- bin/logagent.js | 1 + config/examples/heroku-elasticsearch.yml | 20 +++++++++ .../output-filter/cloudfoundry-format.js | 3 +- lib/plugins/output-filter/heroku-format.js | 42 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 config/examples/heroku-elasticsearch.yml create mode 100644 lib/plugins/output-filter/heroku-format.js diff --git a/bin/logagent.js b/bin/logagent.js index 9b0b9e60..cf2388ee 100755 --- a/bin/logagent.js +++ b/bin/logagent.js @@ -76,6 +76,7 @@ var moduleAlias = { sql: '../lib/plugins/output-filter/sql.js', 'access-watch': '../lib/plugins/output-filter/access-watch.js', 'cloudfoundry-format': '../lib/plugins/output-filter/cloudfoundry-format.js', + 'heroku-format': '../lib/plugins/output-filter/heroku-format.js', 'hash-fields': '../lib/plugins/output-filter/hash-fields.js', 'aes-encrypt-fields': '../lib/plugins/output-filter/aes-encrypt-fields.js', 'ip-truncate-fields': '../lib/plugins/output-filter/ip-truncate-fields.js', diff --git a/config/examples/heroku-elasticsearch.yml b/config/examples/heroku-elasticsearch.yml new file mode 100644 index 00000000..e24c7f10 --- /dev/null +++ b/config/examples/heroku-elasticsearch.yml @@ -0,0 +1,20 @@ +# Global options +options: + # includeOriginalLine: true + # debug: true + +input: + cloudFoundry: + module: input-heroku + port: 9999 + +outputFilter: + heroku-format: + module: heroku-format + +output: + elasticsearch: + module: elasticsearch + url: https://logsene-receiver.sematext.com + # debug: true + \ No newline at end of file diff --git a/lib/plugins/output-filter/cloudfoundry-format.js b/lib/plugins/output-filter/cloudfoundry-format.js index 7895a59d..16b18e96 100644 --- a/lib/plugins/output-filter/cloudfoundry-format.js +++ b/lib/plugins/output-filter/cloudfoundry-format.js @@ -32,7 +32,6 @@ function filterCloudFoundryMessage (data, context) { } function parseCloudFoundryTags (tags) { - console.log(tags) try { const parsedTags2dArr = tags .split(' ') @@ -53,7 +52,7 @@ function parseCloudFoundryTags (tags) { return parsedTags } catch (error) { - console.log(error) + console.error(error) return tags } } diff --git a/lib/plugins/output-filter/heroku-format.js b/lib/plugins/output-filter/heroku-format.js new file mode 100644 index 00000000..6f3ab916 --- /dev/null +++ b/lib/plugins/output-filter/heroku-format.js @@ -0,0 +1,42 @@ +function jsonParse (text) { + try { + return JSON.parse(text) + } catch (err) { + return null + } +} + +function parseHerokuMessage (data, context) { + if (data) { + data._type = context.sourceName.replace('_' + context.index, '') + data.logSource = ('' + data.logSource).replace('_' + context.index, '') + if (!data['@timestamp']) { + data['@timestamp'] = new Date() + } + if (data.message) { + data.json = jsonParse(data.message) + } + } + return data +} + +module.exports = function (context, config, eventEmitter, log, callback) { + try { + const parsedLog = parseHerokuMessage(log, context) + const json = parsedLog.json + const message = parsedLog.message + delete parsedLog.json + delete parsedLog.message + + const structuredLog = { + message, + ...json, + ...parsedLog + } + + return callback(null, structuredLog) + } catch (err) { + console.error(err) + return callback(null, log) + } +} From 020d3f0c85e2a0ea5763215d6bdada604a6d1f57 Mon Sep 17 00:00:00 2001 From: Adnan Rahic Date: Fri, 27 Nov 2020 08:56:38 +0100 Subject: [PATCH 2/5] added json parsing heroku filter --- lib/plugins/output-filter/heroku-format.js | 35 ++++++---------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/lib/plugins/output-filter/heroku-format.js b/lib/plugins/output-filter/heroku-format.js index 6f3ab916..017d92a6 100644 --- a/lib/plugins/output-filter/heroku-format.js +++ b/lib/plugins/output-filter/heroku-format.js @@ -1,20 +1,13 @@ -function jsonParse (text) { - try { - return JSON.parse(text) - } catch (err) { - return null - } -} - function parseHerokuMessage (data, context) { if (data) { - data._type = context.sourceName.replace('_' + context.index, '') - data.logSource = ('' + data.logSource).replace('_' + context.index, '') - if (!data['@timestamp']) { - data['@timestamp'] = new Date() - } - if (data.message) { - data.json = jsonParse(data.message) + const json = data.json + const message = data.message + delete data.json + delete data.message + return { + message, + ...json, + ...data } } return data @@ -23,18 +16,8 @@ function parseHerokuMessage (data, context) { module.exports = function (context, config, eventEmitter, log, callback) { try { const parsedLog = parseHerokuMessage(log, context) - const json = parsedLog.json - const message = parsedLog.message - delete parsedLog.json - delete parsedLog.message - - const structuredLog = { - message, - ...json, - ...parsedLog - } - return callback(null, structuredLog) + return callback(null, parsedLog) } catch (err) { console.error(err) return callback(null, log) From c19f87c2119365ed365f8e7a8f9fc2e70a603867 Mon Sep 17 00:00:00 2001 From: Adnan Rahic Date: Fri, 27 Nov 2020 12:23:00 +0100 Subject: [PATCH 3/5] removed filter from heroku input plugin --- lib/plugins/input/heroku.js | 60 +------------------------------------ 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/lib/plugins/input/heroku.js b/lib/plugins/input/heroku.js index 97d76034..36bad386 100644 --- a/lib/plugins/input/heroku.js +++ b/lib/plugins/input/heroku.js @@ -4,25 +4,6 @@ const extractTokenRegEx = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9 const tokenFormatRegEx = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ const TokenBlacklist = require('../../util/token-blacklist.js') -function jsonParse (text) { - try { - return JSON.parse(text) - } catch (err) { - return null - } -} - -function extractJson (line, source) { - var parsed = {} - if (/^\[{0,1}\{.*\}]{0,1}$/.test(line)) { - parsed = jsonParse(line) - if (!parsed) { - return null - } - return parsed - } -} - function InputHeroku (config, eventEmitter) { this.config = config this.eventEmitter = eventEmitter @@ -79,44 +60,6 @@ InputHeroku.prototype.getHttpServer = function (aport, handler) { } } -function filterHerokuMessage (data, context) { - if (data) { - data._type = context.sourceName.replace('_' + context.index, '') - data.logSource = ('' + data.logSource).replace('_' + context.index, '') - var msg = { - message: data.message, - app: data.app, - host: data.host, - process_type: data.process_type, - originalLine: data.originalLine, - severity: data.severity, - facility: data.facility - } - msg.json = extractJson(msg.message) - var optionalFields = [ - 'method', - 'path', - 'host', - 'request_id', - 'fwd', - 'dyno', - 'connect', - 'service', - 'status', - 'bytes' - ] - optionalFields.forEach(function (f) { - if (data[f]) { - msg[f] = data[f] - } - }) - if (!data['@timestamp']) { - msg['@timestamp'] = new Date() - } - return msg - } -} - InputHeroku.prototype.herokuHandler = function (req, res) { try { var self = this @@ -166,8 +109,7 @@ InputHeroku.prototype.herokuHandler = function (req, res) { } self.eventEmitter.emit('data.raw', line, { sourceName: 'heroku_' + token, - index: token, - filter: filterHerokuMessage + index: token }) }) res.end('ok\n') From 176c6a36f52dd8b0e63ce5d8e1b4c2890959366f Mon Sep 17 00:00:00 2001 From: Adnan Rahic Date: Fri, 27 Nov 2020 12:23:15 +0100 Subject: [PATCH 4/5] added heroku output filter --- lib/plugins/output-filter/heroku-format.js | 73 ++++++++++++++++++---- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/lib/plugins/output-filter/heroku-format.js b/lib/plugins/output-filter/heroku-format.js index 017d92a6..06ddde51 100644 --- a/lib/plugins/output-filter/heroku-format.js +++ b/lib/plugins/output-filter/heroku-format.js @@ -1,25 +1,76 @@ -function parseHerokuMessage (data, context) { +function jsonParse (text) { + try { + return JSON.parse(text) + } catch (err) { + return null + } +} + +function extractJson (line, source) { + let parsed = {} + if (/^\[{0,1}\{.*\}]{0,1}$/.test(line)) { + parsed = jsonParse(line) + if (!parsed) { + return null + } + return parsed + } +} + +function filterHerokuMessage (data, context) { if (data) { - const json = data.json - const message = data.message - delete data.json - delete data.message + data._type = context.sourceName.replace('_' + context.index, '') + data.logSource = ('' + data.logSource).replace('_' + context.index, '') + const msg = { + message: data.message, + app: data.app, + host: data.host, + process_type: data.process_type, + originalLine: data.originalLine, + severity: data.severity, + facility: data.facility + } + + const optionalFields = [ + 'method', + 'path', + 'host', + 'request_id', + 'fwd', + 'dyno', + 'connect', + 'service', + 'status', + 'bytes' + ] + optionalFields.forEach(function (f) { + if (data[f]) { + msg[f] = data[f] + } + }) + if (!data['@timestamp']) { + msg['@timestamp'] = new Date() + } + + const json = extractJson(msg.message) + if (json && json.message) { + delete msg.message + } + return { - message, - ...json, - ...data + ...msg, + ...json } } - return data } module.exports = function (context, config, eventEmitter, log, callback) { try { - const parsedLog = parseHerokuMessage(log, context) + const parsedLog = filterHerokuMessage(log, context) return callback(null, parsedLog) } catch (err) { - console.error(err) + console.log(err) return callback(null, log) } } From afefa731e5198bed2de6664606fe73f1464857f0 Mon Sep 17 00:00:00 2001 From: Adnan Rahic Date: Fri, 27 Nov 2020 12:23:30 +0100 Subject: [PATCH 5/5] added yml config for heroku output filter --- config/examples/heroku-stdout-yml.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/examples/heroku-stdout-yml.yml b/config/examples/heroku-stdout-yml.yml index 1864d4f6..d27f2ae8 100644 --- a/config/examples/heroku-stdout-yml.yml +++ b/config/examples/heroku-stdout-yml.yml @@ -74,6 +74,11 @@ parser: } } } + +outputFilter: + heroku-format: + module: heroku-format + output: # print parsed logs in YAML format to stdout if supress is set false stdout: yaml # use 'pretty' for pretty json and 'ldjson' for line delimited json (default)