-
Notifications
You must be signed in to change notification settings - Fork 79
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #253 from sematext/geo-refactor
- Loading branch information
Showing
6 changed files
with
88 additions
and
98 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| # Global options | ||
| options: | ||
| debug: true | ||
| # geoipEnabled: true | ||
| # geoipField: client_ip | ||
|
|
||
| input: | ||
| files: | ||
| - /var/log/nginx | ||
|
|
||
| parser: | ||
| patterns: | ||
| - | ||
| sourceName: !!js/regexp /.*/ | ||
| match: | ||
| - type: web | ||
| regex: !!js/regexp ^([0-9a-f.:]+)\s[-|\S+]\s(.*) | ||
| fields: | ||
| - client_ip:string | ||
| - message:string | ||
|
|
||
| # outputFilter: | ||
| # geoip: | ||
| # module: geoip | ||
| # field: client_ip | ||
| # debug: true | ||
|
|
||
| output: | ||
| # stdout: yaml | ||
|
|
||
| elasticsearch: | ||
| module: elasticsearch | ||
| url: https://logsene-receiver.sematext.com | ||
| index: token |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,105 +1,55 @@ | ||
| var path = require('path') | ||
| var fs = require('fs') | ||
| var consoleLogger = require('../../util/logger.js') | ||
| var maxmindUpdate = require('../../parser/maxmind-update') | ||
| var MaxMindReader = require('maxmind').Reader | ||
| var cityLookup = null | ||
| var geoIpStatus = { | ||
| maxmindDbDir: process.env.MAXMIND_DB_DIR || '/tmp/', | ||
| const publicIp = require('public-ip') | ||
| const consoleLogger = require('../../util/logger.js') | ||
| const geoIpStatus = { | ||
| debug: !!process.env.DEBUG | ||
| } | ||
|
|
||
| function initMaxmind () { | ||
| maxmindUpdate(true, geoIpStatus.maxmindDbDir, function (err, fileName) { | ||
| if (geoIpStatus.debug) { | ||
| consoleLogger.debug('GeoIP: init GeoIP DB ' + fileName) | ||
| } | ||
| if (err) { | ||
| function initIpAddress () { | ||
| publicIp.v4() | ||
| .then(ip => { | ||
| consoleLogger.log('GeoIP: Loading Public IP') | ||
| geoIpStatus.ipAddress = ip | ||
| }) | ||
| .catch(err => { | ||
| consoleLogger.error('GeoIP: error ' + err) | ||
| geoIpStatus.geoIpFailed = true | ||
| return | ||
| } | ||
| try { | ||
| fs.statSync(fileName) | ||
| cityLookup = new MaxMindReader(fs.readFileSync(fileName), { | ||
| watchForUpdates: true | ||
| }) | ||
| if (cityLookup) { | ||
| consoleLogger.debug('GeoIP: open ' + fileName) | ||
| } else { | ||
| consoleLogger.log('GeoIP: error ' + err) | ||
| } | ||
| if (geoIpStatus.fields) { | ||
| consoleLogger.log( | ||
| 'GeoIP: lookup enabled for fields ' + geoIpStatus.fields | ||
| ) | ||
| } else { | ||
| consoleLogger.log('GeoIP: lookup enabled ') | ||
| } | ||
| } catch (fsStatError) { | ||
| geoIpStatus.geoIpFailed = true | ||
| consoleLogger.error('GeoIP: error ' + fsStatError) | ||
| } | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| function geoipLookup (parsedObject, fieldName, outputFieldName) { | ||
| if (cityLookup != null && parsedObject[fieldName]) { | ||
| var location = cityLookup.get(parsedObject[fieldName]) | ||
| if ( | ||
| location && | ||
| location.country && | ||
| location.city && | ||
| location.city.names && | ||
| location.city.names.en | ||
| ) { | ||
| parsedObject[outputFieldName || 'geoip'] = { | ||
| location: [location.location.longitude, location.location.latitude], | ||
| info: { | ||
| country: location.country.iso_code, | ||
| continent: location.continent.code, | ||
| city: location.city.names.en | ||
| } | ||
| } | ||
| function geoipOutputFilter (context, config, eventEmitter, data, callback) { | ||
| geoIpStatus.debug = config.debug | ||
| geoIpStatus.field = process.env.GEOIP_FIELD || geoIpStatus.field || config.field | ||
|
|
||
| if (geoIpStatus.field) { | ||
| if (geoIpStatus.debug) { | ||
| consoleLogger.log('GeoIP: Lookup enabled for field: ' + geoIpStatus.field) | ||
| } | ||
| } else { | ||
| return null | ||
| } | ||
| } | ||
|
|
||
| function geoipOutputFilter (context, config, eventEmitter, data, callback) { | ||
| // get config values from output-filter config | ||
| if (!geoIpStatus.fields) { | ||
| if (process.env.GEOIP_FIELDS) { | ||
| geoIpStatus.fields = process.env.GEOIP_FIELDS.replace(/\s/g, '').split( | ||
| ',' | ||
| ) | ||
| if (data[geoIpStatus.field]) { | ||
| data.geo = { | ||
| ip: data[geoIpStatus.field] | ||
| } | ||
| } | ||
| geoIpStatus.fields = geoIpStatus.fields || config.fields || ['client_ip'] | ||
| geoIpStatus.debug = config.debug | ||
| consoleLogger.log('GeoIP: lookup enabled for fields ' + geoIpStatus.fields) | ||
| return callback(null, data) | ||
| } | ||
|
|
||
| if (geoIpStatus.geoIpFailed === true) { | ||
| if (geoIpStatus.debug) { | ||
| console.log('GeoIP: Lookup failed for the Public IP address') | ||
| } | ||
|
|
||
| return callback(null, data) | ||
| } | ||
| for (var i = 0; i < geoIpStatus.fields.length; i++) { | ||
| if (data[geoIpStatus.fields[i]] !== undefined) { | ||
| geoipLookup(data, geoIpStatus.fields[i], 'geoip') | ||
| } | ||
|
|
||
| if (geoIpStatus.debug) { | ||
| console.log(`GeoIP: The Public IP address is ${geoIpStatus.ipAddress}`) | ||
| } | ||
| data.geo = { | ||
| ip: geoIpStatus.ipAddress | ||
| } | ||
| return callback(null, data) | ||
| } | ||
|
|
||
| /* | ||
| function test () { | ||
| var cfg = { fields: ['client_ip'], maxmindDbDir: '/tmp/', debug: true } | ||
| setInterval(() => { | ||
| geoipOutputFilter(null, cfg, null, { client_ip: '94.216.99.1'}, console.log) | ||
| }, 200) | ||
| } | ||
| */ | ||
|
|
||
| consoleLogger.log('GeoIP: Loading Maxmind DB ') | ||
| initMaxmind() | ||
| initIpAddress() | ||
|
|
||
| module.exports = geoipOutputFilter |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters