added gzip

sematext · Dec 8, 2020 · e687ee9 · e687ee9
1 parent c101541
commit e687ee9
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 20 deletions.
diff --git a/config/examples/aws-ecs-input-es-output.yml b/config/examples/aws-ecs-input-es-output.yml
@@ -5,7 +5,7 @@ input:
     module: input-aws-ecs
     port: 6666
     useIndexFromUrlPath: true
-    workers: 4
+    # workers: 4
 
 outputFilter: 
   aws-ecs-format: 
@@ -16,3 +16,4 @@ output:
   sematext-cloud:
     module: elasticsearch
     url: https://logsene-receiver.sematext.com
+    # debug: true
diff --git a/lib/plugins/input/aws-ecs.js b/lib/plugins/input/aws-ecs.js
@@ -4,6 +4,7 @@ const throng = require('throng')
 const extractTokenRegEx = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/
 const tokenFormatRegEx = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
 const TokenBlacklist = require('../../util/token-blacklist.js')
+const zlib = require('zlib')
 
 class AwsEcs {
   constructor (config, eventEmitter) {
@@ -97,7 +98,6 @@ class AwsEcs {
     if (body.length === 0) {
       return
     }
-
     const self = this
     const docs = JSON.parse(body)
     if (docs && docs.length > 0) {
@@ -126,7 +126,9 @@ class AwsEcs {
       const self = this
       const path = req.url.split('/')
       let token = null
-      let bodyIn = ''
+      // let bodyIn = ''
+      const buffer = []
+
       if (self.config.useIndexFromUrlPath === true && path.length > 1) {
         if (path[1] && path[1].length > 31 && tokenFormatRegEx.test(path[1])) {
           const match = path[1].match(extractTokenRegEx)
@@ -147,25 +149,52 @@ class AwsEcs {
         res.end(`invalid logs token in url ${req.url}`)
         return
       }
-      req.on('data', function (data) {
-        bodyIn += String(data)
-      })
-      req.on('end', function endHandler () {
-        try {
-          self.parseRes(req.headers, bodyIn, token)
-        } catch (err) {
-          if (self.config.debug) {
-            consoleLogger.error('Error in ECS HttpHandler: ' + err)
+
+      const isGzip = req.headers['content-encoding'] === 'gzip'
+
+      if (isGzip) {
+        console.log(isGzip)
+        console.log('\n')
+        console.log('\n')
+        console.log('\n')
+
+        const gunzip = zlib.createGunzip()
+        req.pipe(gunzip)
+
+        gunzip.on('data', function (data) {
+          buffer.push(data.toString())
+        })
+        gunzip.on('end', function endHandler () {
+          try {
+            // console.log(buffer)
+            // console.log(typeof buffer)
+            // console.log('\n')
+
+            const body = buffer.join('')
+
+            // console.log(final)
+            // console.log(typeof final)
+            // console.log('\n')
+
+            self.parseRes(req.headers, body, token)
+          } catch (err) {
+            if (self.config.debug) {
+              consoleLogger.error('Error in ECS HttpHandler: ' + err)
+            }
+
+            res.writeHead(500, { 'Content-Type': 'text/plain' })
+            res.end(`Invalid json input: ${err}\n`)
+            return
           }
+          // send response to client
+          res.writeHead(200, { 'Content-Type': 'text/plain' })
+          res.end('OK\n')
+        })
 
-          res.writeHead(500, { 'Content-Type': 'text/plain' })
-          res.end(`Invalid json input: ${err}\n`)
-          return
-        }
-        // send response to client
-        res.writeHead(200, { 'Content-Type': 'text/plain' })
-        res.end('OK\n')
-      })
+        return
+      } else {
+
+      }
     } catch (err) {
       res.statusCode = 500
       res.end()