added input filter for device detection and parsing of userAgent head…

…ers for web logs

bin/logagent.js

@@ -62,6 +62,7 @@ var moduleAlias = {
   'azure-event-hub': '../lib/plugins/input/azure-event-hub.js',
   'unix-socket-reader': '../lib/plugins/input/unixSocketReader.js',
   // input filters
+  'httpDeviceDetector': '../lib/plugins/input-filter/httpDeviceDetector.js',
   'input-filter-k8s-containerd': '../lib/plugins/input-filter/kubernetesContainerd.js',
   'grep': '../lib/plugins/input-filter/grep.js',
   'grok': 'logagent-input-filter-grok',

config/examples/file-input-filter-useragent-es-output.yml

@@ -0,0 +1,24 @@
+# Global options
+options:
+  includeOriginalLine: false
+  printStats: 10
+  maxInputRate: 1mb # per second
+
+input:
+  stdin: true
+  files:
+      - '/var/log/**/*.log'
+
+inputFilter:
+  - module: httpDeviceDetector
+    # optional setting to configure the field name for the useragent
+    # will default to 'useragent' if this config is omitted
+    # config:
+    #   userAgentFieldName: useragent
+
+output:
+  # stdout: yaml  
+  sematext-logs:
+    module: elasticsearch
+    url: https://logsene-receiver.sematext.com
+    index: LOGS_TOKEN

lib/plugins/input-filter/httpDeviceDetector.js

@@ -0,0 +1,25 @@
+const DeviceDetector = require('device-detector-js')
+const deviceDetector = new DeviceDetector()
+var safeStringify = require('fast-safe-stringify')
+function jsonParse (text) {
+  try {
+    return JSON.parse(text)
+  } catch (err) {
+    return null
+  }
+}
+
+module.exports = function (context, config, data, callback) {
+  try {
+    const userAgentFieldName = config.userAgentFieldName || 'useragent'
+    const parsedData = jsonParse(data)
+    const useragent = parsedData[userAgentFieldName]
+
+    const device = deviceDetector.parse(useragent)
+    const dataWithUseragentDetails = { ...device, ...parsedData }
+
+    return callback(null, safeStringify(dataWithUseragentDetails))
+  } catch (err) {
+    return callback(null, data)
+  }
+}

package.json

@@ -83,6 +83,7 @@
     "clone": "^2.1.2",
     "co": "^4.6.0",
     "commander": "^2.19.0",
+    "device-detector-js": "^2.2.1",
     "docker-events": "0.0.2",
     "docker-loghose": "^1.6.5",
     "dockerode": "^2.5.6",