Add support for GeoIPLite2-ASN database (logstash-plugins#120)

* Handle ASN. this means we need to use a new Java driver.

Author: Suyog Rao

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.2.0
+  - Add support for GeoLite2-ASN database from Maxmind for ASN data.
+  - Update Java dependencies to 2.9.0 to support the new ASN database.
+
 ## 4.1.1
   - Add support for commercial databases from Maxmind.
   - Add ASN data support via GeoIP2-ISP database.

build.gradle

@@ -1,3 +1,5 @@
+import de.undercouch.gradle.tasks.download.Download
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -16,14 +18,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 apply plugin: "java"
 apply plugin: "distribution"
 apply plugin: "idea"
 
 // TODO(sissel): Move this to a file shared by the gemspec.
 group "org.logstash.filters"
-version "4.1.0"
+version "4.2.0"
 
 import java.nio.file.Files
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING
@@ -35,6 +36,7 @@ buildscript {
 
   dependencies {
     classpath group: 'org.jruby', name: 'jruby-complete', version: "1.7.26"
+    classpath 'de.undercouch:gradle-download-task:3.2.0'
   }
 }
 
@@ -45,18 +47,18 @@ repositories {
 dependencies {
   compileOnly group: "org.apache.logging.log4j", name: "log4j-api", version: "2.6.2"
   compileOnly group: "org.apache.logging.log4j", name: "log4j-core", version: "2.6.2"
-  compileOnly group: "com.maxmind.geoip2", name: "geoip2", version: "2.8.0"
-  compileOnly group: "com.maxmind.db", name: "maxmind-db", version: "1.2.1"
+  compileOnly group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0"
+  compileOnly group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2"
 
-  runtime group: "com.maxmind.geoip2", name: "geoip2", version: "2.8.0"
-  runtime group: "com.maxmind.db", name: "maxmind-db", version: "1.2.1"
+  runtime group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0"
+  runtime group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2"
 
   testCompile group: 'junit', name: 'junit', version: '4.12'
   testCompile group: "org.apache.logging.log4j", name: "log4j-api", version: "2.6.2"
   testCompile group: "org.apache.logging.log4j", name: "log4j-core", version: "2.6.2"
   testCompile group: 'org.jruby', name: 'jruby-complete', version: "1.7.26"
-  testCompile group: "com.maxmind.geoip2", name: "geoip2", version: "2.8.0"
-  testCompile group: "com.maxmind.db", name: "maxmind-db", version: "1.2.1"
+  testCompile group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0"
+  testCompile group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2"
   testCompile fileTree(dir: logstashCoreGemPath, include: '**/*.jar')
   testCompile fileTree(dir: logstashCoreEventGemPath, include: '**/*.jar')
 
@@ -65,6 +67,10 @@ dependencies {
   compileOnly fileTree(dir: logstashCoreEventGemPath, include: '**/*.jar')
 }
 
+test {
+  dependsOn 'downloadASNDatabaseForTest'
+}
+
 task rubyBootstrap << {
   description "Try bundler"
   def jruby = new org.jruby.embed.ScriptingContainer()
@@ -120,4 +126,19 @@ task vendor << {
   Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING)
 }
 
+String databaseRootResource = 'http://geolite.maxmind.com/download/geoip/database/'
+
+task downloadASNDatabaseForTest(type: Download, overwrite: false) {
+  src([
+    databaseRootResource + 'GeoLite2-ASN.tar.gz'
+  ])
+  dest file("build")
+  doLast {
+    copy {
+      from tarTree('build/GeoLite2-ASN.tar.gz')
+      into file("build")
+    }
+  }
+}
+
 vendor.dependsOn(jar, generateGemJarRequiresFile)

ci/run.sh

@@ -1,3 +1,6 @@
+#!/bin/bash
+current_dir="$(dirname "$0")"
+
 bundle install
 bundle exec rake gradle.properties
 ./gradlew assemble

docs/index.asciidoc

@@ -21,8 +21,23 @@ include::{include_path}/plugin_header.asciidoc[]
 ==== Description
 
 The GeoIP filter adds information about the geographical location of IP addresses,
-based on data from the Maxmind GeoLite2 database. Commercial databases from Maxmind are 
-also supported in this plugin.
+based on data from the Maxmind GeoLite2 databases.
+
+==== Supported Databases
+
+This plugin is bundled with https://dev.maxmind.com/geoip/geoip2/geolite2[GeoLite2] City database out of the box. From Maxmind's description -- 
+"GeoLite2 databases are free IP geolocation databases comparable to, but less accurate than, MaxMind’s 
+GeoIP2 databases". Please see GeoIP Lite2 license for more details.
+
+https://www.maxmind.com/en/geoip2-databases[Commercial databases] from Maxmind are also supported in this plugin.
+
+If you need to use databases other than the bundled GeoLite2 City, you can download them directly 
+from Maxmind's website and use the `database` option to specify their location. The GeoLite2 databases 
+can be downloaded from https://dev.maxmind.com/geoip/geoip2/geolite2[here].
+
+If you would like to get Autonomous System Number(ASN) information, you can use the GeoLite2-ASN database.
+
+==== Details
 
 A `[geoip][location]` field is created if
 the GeoIP lookup returns a latitude and longitude. The field is stored in

lib/logstash-filter-geoip_jars.rb

@@ -1,6 +1,6 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('com.maxmind.geoip2', 'geoip2', '2.8.0')
-require_jar('com.maxmind.db', 'maxmind-db', '1.2.1')
-require_jar('org.logstash.filters', 'logstash-filter-geoip', '4.1.0')
+require_jar('com.maxmind.geoip2', 'geoip2', '2.9.0')
+require_jar('com.maxmind.db', 'maxmind-db', '1.2.2')
+require_jar('org.logstash.filters', 'logstash-filter-geoip', '4.2.0')

logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '4.1.1'
+  s.version         = '4.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "$summary"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/filters/geoip_spec.rb

@@ -3,6 +3,8 @@
 require "logstash/filters/geoip"
 
 CITYDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/GeoLite2-City.mmdb").first
+# this is downloaded in build dir so we don't accidentally package this database when creating a gem
+ASNDB = ::Dir.glob(::File.expand_path("../../build/GeoLite2-ASN_*", ::File.dirname(__FILE__))+"/GeoLite2-ASN.mmdb").first
 
 describe LogStash::Filters::GeoIP do
 
@@ -266,4 +268,23 @@
     end
   end
 
+  describe "GeoIP2-ASN database" do
+    config <<-CONFIG
+      filter {
+        geoip {
+          source => "ip"
+          database => "#{ASNDB}"
+        }
+      }
+    CONFIG
+
+    sample("ip" => "8.8.8.8") do
+      expect(subject.get("geoip")).not_to be_empty
+      expect(subject.get("geoip")["asn"]).to eq(15169)
+      expect(subject.get("geoip")["as_org"]).to eq("Google Inc.")
+    end
+
+
+  end
+
 end

src/main/java/org/logstash/filters/Fields.java

@@ -67,6 +67,9 @@ public String fieldName() {
   static final EnumSet<Fields> DEFAULT_ISP_FIELDS = EnumSet.of(Fields.IP, Fields.AUTONOMOUS_SYSTEM_NUMBER,
       Fields.AUTONOMOUS_SYSTEM_ORGANIZATION, Fields.ISP, Fields.ORGANIZATION);
 
+  static final EnumSet<Fields> DEFAULT_ASN_LITE_FIELDS = EnumSet.of(Fields.IP, Fields.AUTONOMOUS_SYSTEM_NUMBER,
+      Fields.AUTONOMOUS_SYSTEM_ORGANIZATION);
+
   public static Fields parseField(String value) {
     try {
       return valueOf(value.toUpperCase(Locale.ROOT));

src/main/java/org/logstash/filters/GeoIPFilter.java

@@ -22,6 +22,7 @@
 import com.maxmind.db.InvalidDatabaseException;
 import com.maxmind.geoip2.exception.AddressNotFoundException;
 import com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.maxmind.geoip2.model.AsnResponse;
 import com.maxmind.geoip2.model.CityResponse;
 import com.maxmind.geoip2.model.CountryResponse;
 import com.maxmind.geoip2.model.IspResponse;
@@ -50,7 +51,7 @@ public class GeoIPFilter {
   private static final String CITY_DB_TYPE = "GeoIP2-City";
   private static final String COUNTRY_DB_TYPE = "GeoIP2-Country";
   private static final String ISP_DB_TYPE = "GeoIP2-ISP";
-  
+
   private final String sourceField;
   private final String targetField;
   private final Set<Fields> desiredFields;
@@ -83,9 +84,11 @@ private Set<Fields> createDesiredFields(List<String> fields) {
           desiredFields = Fields.DEFAULT_COUNTRY_FIELDS;
           break;
         case ISP_DB_TYPE:
-        case ASN_LITE_DB_TYPE:
           desiredFields = Fields.DEFAULT_ISP_FIELDS;
           break;
+        case ASN_LITE_DB_TYPE:
+          desiredFields = Fields.DEFAULT_ASN_LITE_FIELDS;
+          break;
       }
     } else {
       for (String fieldName : fields) {
@@ -126,6 +129,8 @@ public boolean handleEvent(RubyEvent rubyEvent) {
           geoData = retrieveCountryGeoData(ipAddress);
           break;
         case ASN_LITE_DB_TYPE:
+          geoData = retrieveAsnGeoData(ipAddress);
+          break;
         case ISP_DB_TYPE:
           geoData = retrieveIspGeoData(ipAddress);
           break;
@@ -351,4 +356,30 @@ private Map<String, Object> retrieveIspGeoData(InetAddress ipAddress) throws Geo
 
     return geoData;
   }
+
+  private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException {
+    AsnResponse response = databaseReader.asn(ipAddress);
+    Map<String, Object> geoData = new HashMap<>();
+    for (Fields desiredField : this.desiredFields) {
+      switch (desiredField) {
+        case IP:
+          geoData.put(Fields.IP.fieldName(), ipAddress.getHostAddress());
+          break;
+        case AUTONOMOUS_SYSTEM_NUMBER:
+          Integer asn = response.getAutonomousSystemNumber();
+          if (asn != null) {
+            geoData.put(Fields.AUTONOMOUS_SYSTEM_NUMBER.fieldName(), asn);
+          }
+          break;
+        case AUTONOMOUS_SYSTEM_ORGANIZATION:
+          String aso = response.getAutonomousSystemOrganization();
+          if (aso != null) {
+            geoData.put(Fields.AUTONOMOUS_SYSTEM_ORGANIZATION.fieldName(), aso);
+          }
+          break;
+      }
+    }
+
+    return geoData;
+  }
 }