diff --git a/config/examples/aws-ecs-input-es-output.yml b/config/examples/aws-ecs-input-es-output.yml index 4e5e366b..4afb7410 100644 --- a/config/examples/aws-ecs-input-es-output.yml +++ b/config/examples/aws-ecs-input-es-output.yml @@ -5,7 +5,7 @@ input: module: input-aws-ecs port: 6666 useIndexFromUrlPath: true - workers: 4 + # workers: 4 outputFilter: aws-ecs-format: @@ -16,3 +16,4 @@ output: sematext-cloud: module: elasticsearch url: https://logsene-receiver.sematext.com + # debug: true diff --git a/lib/plugins/input/aws-ecs.js b/lib/plugins/input/aws-ecs.js index fe16f98d..f660ee90 100644 --- a/lib/plugins/input/aws-ecs.js +++ b/lib/plugins/input/aws-ecs.js @@ -4,6 +4,7 @@ const throng = require('throng') const extractTokenRegEx = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/ const tokenFormatRegEx = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ const TokenBlacklist = require('../../util/token-blacklist.js') +const zlib = require('zlib') class AwsEcs { constructor (config, eventEmitter) { @@ -97,7 +98,6 @@ class AwsEcs { if (body.length === 0) { return } - const self = this const docs = JSON.parse(body) if (docs && docs.length > 0) { @@ -126,7 +126,9 @@ class AwsEcs { const self = this const path = req.url.split('/') let token = null - let bodyIn = '' + // let bodyIn = '' + const buffer = [] + if (self.config.useIndexFromUrlPath === true && path.length > 1) { if (path[1] && path[1].length > 31 && tokenFormatRegEx.test(path[1])) { const match = path[1].match(extractTokenRegEx) @@ -147,25 +149,52 @@ class AwsEcs { res.end(`invalid logs token in url ${req.url}`) return } - req.on('data', function (data) { - bodyIn += String(data) - }) - req.on('end', function endHandler () { - try { - self.parseRes(req.headers, bodyIn, token) - } catch (err) { - if (self.config.debug) { - consoleLogger.error('Error in ECS HttpHandler: ' + err) + + const isGzip = req.headers['content-encoding'] === 'gzip' + + if (isGzip) { + console.log(isGzip) + console.log('\n') + console.log('\n') + console.log('\n') + + const gunzip = zlib.createGunzip() + req.pipe(gunzip) + + gunzip.on('data', function (data) { + buffer.push(data.toString()) + }) + gunzip.on('end', function endHandler () { + try { + // console.log(buffer) + // console.log(typeof buffer) + // console.log('\n') + + const body = buffer.join('') + + // console.log(final) + // console.log(typeof final) + // console.log('\n') + + self.parseRes(req.headers, body, token) + } catch (err) { + if (self.config.debug) { + consoleLogger.error('Error in ECS HttpHandler: ' + err) + } + + res.writeHead(500, { 'Content-Type': 'text/plain' }) + res.end(`Invalid json input: ${err}\n`) + return } + // send response to client + res.writeHead(200, { 'Content-Type': 'text/plain' }) + res.end('OK\n') + }) - res.writeHead(500, { 'Content-Type': 'text/plain' }) - res.end(`Invalid json input: ${err}\n`) - return - } - // send response to client - res.writeHead(200, { 'Content-Type': 'text/plain' }) - res.end('OK\n') - }) + return + } else { + + } } catch (err) { res.statusCode = 500 res.end()