elastic · martijnvg · Oct 13, 2021 · Sep 28, 2021 · Oct 7, 2021 · Oct 7, 2021
diff --git a/docs/build.gradle b/docs/build.gradle
@@ -63,8 +63,6 @@ testClusters.matching { it.name == "integTest"}.configureEach {
   if (singleNode().testDistribution == DEFAULT) {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
-    setting 'ingest.geoip.downloader.enabled', 'false'
-    systemProperty 'es.geoip_v2_feature_flag_enabled', 'true'
     keystorePassword 'keystore-password'
   }
 
@@ -89,6 +87,9 @@ testClusters.matching { it.name == "integTest"}.configureEach {
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 
+  extraConfigFile 'ingest-geoip/GeoLite2-City.mmdb', file("${project.projectDir}/src/test/resources/GeoLite2-City.mmdb")
+  extraConfigFile 'ingest-geoip/GeoLite2-Country.mmdb', file("${project.projectDir}/src/test/resources/GeoLite2-Country.mmdb")
+
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
@@ -114,6 +115,10 @@ tasks.named("integTest").configure {
   }
 }
 
+tasks.named("forbiddenPatterns").configure {
+  exclude '**/*.mmdb'
+}
+
 tasks.named("buildRestTests").configure {
   docs = docsFileTree
 }

diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc
@@ -175,7 +175,7 @@ PUT _index_template/my-data-stream-template
 ----
 POST my-data-stream/_doc?pipeline=my-pipeline
 {
-  "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\""
+  "message": "89.160.20.128 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\""
 }
 ----
 // TEST[s/my-pipeline/my-pipeline&refresh=wait_for/]
@@ -216,21 +216,21 @@ The API returns:
             "version": "1.1"
           },
           "source": {
-            "ip": "212.87.37.154",
+            "ip": "89.160.20.128",
             "geo": {
-              "continent_name": "Europe",
-              "region_iso_code": "DE-BE",
-              "city_name": "Berlin",
-              "country_iso_code": "DE",
-              "country_name": "Germany",
-              "region_name": "Land Berlin",
-              "location": {
-                "lon": 13.4978,
-                "lat": 52.411
+              "continent_name" : "Europe",
+              "country_name" : "Sweden",
+              "country_iso_code" : "SE",
+              "city_name" : "Linköping",
+              "region_iso_code" : "SE-E",
+              "region_name" : "Östergötland County",
+              "location" : {
+                "lon" : 15.6167,
+                "lat" : 58.4167
               }
             }
           },
-          "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"",
+          "message": "89.160.20.128 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"",
           "url": {
             "original": "/favicon.ico"
           },

diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
@@ -69,7 +69,7 @@ PUT _ingest/pipeline/geoip
 }
 PUT my-index-000001/_doc/my_id?pipeline=geoip
 {
-  "ip": "8.8.8.8"
+  "ip": "89.160.20.128"
 }
 GET my-index-000001/_doc/my_id
 --------------------------------------------------
@@ -86,12 +86,15 @@ Which returns:
   "_seq_no": 55,
   "_primary_term": 1,
   "_source": {
-    "ip": "8.8.8.8",
+    "ip": "89.160.20.128",
     "geoip": {
-      "continent_name": "North America",
-      "country_name": "United States",
-      "country_iso_code": "US",
-      "location": { "lat": 37.751, "lon": -97.822 }
+      "continent_name": "Europe",
+      "country_name": "Sweden",
+      "country_iso_code": "SE",
+      "city_name" : "Linköping",
+      "region_iso_code" : "SE-E",
+      "region_name" : "Östergötland County",
+      "location": { "lat": 58.4167, "lon": 15.6167 }
     }
   }
 }
@@ -119,7 +122,7 @@ PUT _ingest/pipeline/geoip
 }
 PUT my-index-000001/_doc/my_id?pipeline=geoip
 {
-  "ip": "8.8.8.8"
+  "ip": "89.160.20.128"
 }
 GET my-index-000001/_doc/my_id
 --------------------------------------------------
@@ -136,11 +139,11 @@ returns this:
   "_seq_no": 65,
   "_primary_term": 1,
   "_source": {
-    "ip": "8.8.8.8",
+    "ip": "89.160.20.128",
     "geo": {
-      "continent_name": "North America",
-      "country_name": "United States",
-      "country_iso_code": "US"
+      "continent_name": "Europe",
+      "country_name": "Sweden",
+      "country_iso_code": "SE"
     }
   }
 }
@@ -236,7 +239,7 @@ PUT _ingest/pipeline/geoip
 
 PUT my_ip_locations/_doc/1?refresh=true&pipeline=geoip
 {
-  "ip": "8.8.8.8"
+  "ip": "89.160.20.128"
 }
 
 GET /my_ip_locations/_search
@@ -250,8 +253,8 @@ GET /my_ip_locations/_search
         "geo_distance": {
           "distance": "1m",
           "geoip.location": {
-            "lon": -97.822,
-            "lat": 37.751
+            "lon": 15.6167,
+            "lat": 58.4167
           }
         }
       }
@@ -285,15 +288,18 @@ GET /my_ip_locations/_search
         "_score" : 1.0,
         "_source" : {
           "geoip" : {
-            "continent_name" : "North America",
-            "country_name" : "United States",
-            "country_iso_code" : "US",
+            "continent_name" : "Europe",
+            "country_name" : "Sweden",
+            "country_iso_code" : "SE",
+            "city_name" : "Linköping",
+            "region_iso_code" : "SE-E",
+            "region_name" : "Östergötland County",
             "location" : {
-              "lon" : -97.822,
-              "lat" : 37.751
+              "lon" : 15.6167,
+              "lat" : 58.4167
             }
           },
-          "ip" : "8.8.8.8"
+          "ip" : "89.160.20.128"
         }
       }
     ]

diff --git a/docs/src/test/resources/GeoLite2-City.mmdb b/docs/src/test/resources/GeoLite2-City.mmdb
diff --git a/docs/src/test/resources/GeoLite2-Country.mmdb b/docs/src/test/resources/GeoLite2-Country.mmdb
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
@@ -13,7 +13,7 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
-  description 'Ingest processor that uses lookup geo data based on IP adresses using the MaxMind geo database'
+  description 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database'
   classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin'
 }
 
@@ -57,14 +57,7 @@ tasks.named("internalClusterTest").configure {
   }
 }
 
-tasks.register("copyDefaultGeoIp2DatabaseFiles", Copy) {
-  from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) }
-  into "${project.buildDir}/ingest-geoip"
-  include "*.mmdb"
-}
-
 tasks.named("bundlePlugin").configure {
-  dependsOn("copyDefaultGeoIp2DatabaseFiles")
   from("${project.buildDir}/ingest-geoip") {
     into '/'
   }
@@ -107,3 +100,18 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /maxmind-db.*/, to: 'maxmind-db-reader'
   ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt'
 }
+
+testClusters.configureEach {
+  // Needed for database downloader, uses delete-by-query to cleanup old databases from org.elasticsearch.ingest.geoip database system index
+  module ':modules:reindex'
+  // Downloader is enabled by default, but in test clusters in build disabled by default,
+  // but in this module, the downloader should be enabled by default
+  systemProperty 'ingest.geoip.downloader.enabled.default', 'true'
+  if (useFixture) {
+    setting 'ingest.geoip.downloader.endpoint', { "${-> fixtureAddress()}" }
+  }
+}
+
+tasks.named("yamlRestTestV7CompatTransform").configure { task ->
+  task.skipTestsByFilePattern("**/ingest_geoip/20_geoip_processor.yml", "from 8.0 yaml rest tests use geoip test fixture and default geoip are no longer packaged. In 7.x yaml tests used default databases which makes tests results very different, so skipping these tests")
+}
diff --git a/.../qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java b/.../qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java
@@ -7,25 +7,25 @@
  */
 package org.elasticsearch.ingest.geoip;
 
-import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.xcontent.ObjectPath;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class UpdateDatabasesIT extends ESRestTestCase {
 
@@ -35,8 +35,14 @@ public void test() throws Exception {
         Request simulatePipelineRequest = new Request("POST", "/_ingest/pipeline/_simulate");
         simulatePipelineRequest.setJsonEntity(body);
         {
-            Map<String, Object> response = toMap(client().performRequest(simulatePipelineRequest));
-            assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Tumba"));
+            Map<String, Object> response = entityAsMap(client().performRequest(simulatePipelineRequest));
+            assertThat(ObjectPath.eval("docs.0.doc._source.tags.0", response), equalTo("_geoip_database_unavailable_GeoLite2-City.mmdb"));
+        }
+
+        // Ensure no config databases have been setup:
+        {
+            Map<?, ?> stats = getGeoIpStatsForSingleNode();
+            assertThat(stats, nullValue());
         }
 
         Path configPath = PathUtils.get(System.getProperty("tests.config.dir"));
@@ -46,14 +52,25 @@ public void test() throws Exception {
         Files.copy(UpdateDatabasesIT.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
             ingestGeoipDatabaseDir.resolve("GeoLite2-City.mmdb"));
 
-        assertBusy(() -> {
-            Map<String, Object> response = toMap(client().performRequest(simulatePipelineRequest));
-            assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping"));
-        });
+        // Ensure that a config database has been setup:
+        {
+            assertBusy(() -> {
+                Map<?, ?> stats = getGeoIpStatsForSingleNode();
+                assertThat(stats, notNullValue());
+                assertThat(stats.get("config_databases"), equalTo(List.of("GeoLite2-City.mmdb")));
+            });
+        }
+
+        Map<String, Object> response = entityAsMap(client().performRequest(simulatePipelineRequest));
+        assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping"));
     }
 
-    private static Map<String, Object> toMap(Response response) throws IOException {
-        return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
+    private static Map<?, ?> getGeoIpStatsForSingleNode() throws IOException {
+        Request request = new Request("GET", "/_ingest/geoip/stats");
+        Map<String, Object> response = entityAsMap(client().performRequest(request));
+        Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
+        assertThat(nodes.size(), either(equalTo(0)).or(equalTo(1)));
+        return nodes.isEmpty() ? null : (Map<?, ?>) nodes.values().iterator().next();
     }
 
     @Override