diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java index ae811db226b06..92ec911dbf451 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java @@ -16,17 +16,14 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.StreamsUtils; import org.junit.ClassRule; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.List; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; + public abstract class AbstractGeoIpIT extends ESIntegTestCase { private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false; @@ -45,23 +42,7 @@ protected Collection> nodePlugins() { @Override protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSettings) { final Path databasePath = createTempDir(); - try { - Files.createDirectories(databasePath); - Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")), - databasePath.resolve("GeoLite2-City.mmdb") - ); - Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")), - databasePath.resolve("GeoLite2-Country.mmdb") - ); - Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")), - databasePath.resolve("GeoLite2-ASN.mmdb") - ); - } catch (final IOException e) { - throw new UncheckedIOException(e); - } + copyDefaultDatabases(databasePath); return Settings.builder() .put("ingest.geoip.database_path", databasePath) .put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index f7ab384c69bf1..d994bd70eb7a0 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -66,6 +66,7 @@ import java.util.zip.GZIPInputStream; import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.anEmptyMap; @@ -688,12 +689,7 @@ private void setupDatabasesInConfigDirectory() throws Exception { .forEach(path -> { try { Files.createDirectories(path); - Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-City.mmdb"), path.resolve("GeoLite2-City.mmdb")); - Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), path.resolve("GeoLite2-ASN.mmdb")); - Files.copy( - GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-Country.mmdb"), - path.resolve("GeoLite2-Country.mmdb") - ); + copyDefaultDatabases(path); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index 8d8b0b4215b3f..87daefab7b428 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -22,10 +22,8 @@ import org.elasticsearch.watcher.ResourceWatcherService; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -34,7 +32,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -68,8 +67,8 @@ public void test() throws Exception { when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir, clusterService); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City.mmdb")); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); + copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb")); + copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb")); databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); lazyLoadReaders(databaseNodeService); @@ -138,18 +137,14 @@ public void test() throws Exception { assertThat(previous1.current(), equalTo(-1)); }); } else { - Files.copy( - ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), - geoIpTmpDir.resolve("GeoLite2-City.mmdb"), - StandardCopyOption.REPLACE_EXISTING - ); + copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb")); databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb")); } DatabaseReaderLazyLoader previous2 = databaseNodeService.get("GeoLite2-City-Test.mmdb"); - InputStream source = ConfigDatabases.class.getResourceAsStream( - i % 2 == 0 ? "/GeoIP2-City-Test.mmdb" : "/GeoLite2-City-Test.mmdb" + copyDatabase( + i % 2 == 0 ? "GeoIP2-City-Test.mmdb" : "GeoLite2-City-Test.mmdb", + geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb") ); - Files.copy(source, geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), StandardCopyOption.REPLACE_EXISTING); databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); DatabaseReaderLazyLoader current1 = databaseNodeService.get("GeoLite2-City.mmdb"); @@ -194,7 +189,7 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI throws IOException { GeoIpCache cache = new GeoIpCache(0); ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache); - copyDatabaseFiles(geoIpConfigDir, configDatabases); + copyDefaultDatabases(geoIpConfigDir, configDatabases); DatabaseNodeService databaseNodeService = new DatabaseNodeService( geoIpTmpDir, mock(Client.class), diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java index 01d7cdc9b9d5c..7b962fed0ca83 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java @@ -20,12 +20,11 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; -import java.nio.file.CopyOption; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -62,8 +61,8 @@ public void testLocalDatabasesEmptyConfig() throws Exception { public void testDatabasesConfigDir() throws Exception { Path configDir = createTempDir(); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb")); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); + copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb")); + copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb")); ConfigDatabases configDatabases = new ConfigDatabases(configDir, new GeoIpCache(0)); configDatabases.initialize(resourceWatcherService); @@ -92,9 +91,9 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception { assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); } - CopyOption option = StandardCopyOption.REPLACE_EXISTING; - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb")); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), option); + copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb")); + copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb")); + assertBusy(() -> { assertThat(configDatabases.getConfigDatabases().size(), equalTo(4)); DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-ASN.mmdb"); @@ -116,7 +115,8 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception { public void testDatabasesUpdateExistingConfDatabase() throws Exception { Path configDir = createTempDir(); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); + copyDatabase("GeoLite2-City.mmdb", configDir); + GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); configDatabases.initialize(resourceWatcherService); @@ -131,11 +131,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { assertThat(cache.count(), equalTo(1)); } - Files.copy( - ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), - configDir.resolve("GeoLite2-City.mmdb"), - StandardCopyOption.REPLACE_EXISTING - ); + copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb")); assertBusy(() -> { assertThat(configDatabases.getConfigDatabases().size(), equalTo(1)); assertThat(cache.count(), equalTo(0)); @@ -154,11 +150,9 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { }); } - private static Path prepareConfigDir() throws IOException { + private static Path prepareConfigDir() { Path dir = createTempDir(); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), dir.resolve("GeoLite2-ASN.mmdb")); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), dir.resolve("GeoLite2-City.mmdb")); - Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-Country.mmdb"), dir.resolve("GeoLite2-Country.mmdb")); + copyDefaultDatabases(dir); return dir; } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java index 34d5429142cec..1579c7020c58a 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java @@ -83,7 +83,7 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask; import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.TYPE; import static org.hamcrest.Matchers.empty; @@ -117,10 +117,9 @@ public class DatabaseNodeServiceTests extends ESTestCase { @Before public void setup() throws IOException { final Path geoIpConfigDir = createTempDir(); - Files.createDirectories(geoIpConfigDir); GeoIpCache cache = new GeoIpCache(1000); ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache); - copyDatabaseFiles(geoIpConfigDir, configDatabases); + copyDefaultDatabases(geoIpConfigDir, configDatabases); threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName()); Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 663ae1152246a..a0541df0d4d8a 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -25,18 +25,15 @@ import org.elasticsearch.ingest.geoip.Database.Property; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; import org.junit.After; import org.junit.Before; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -45,6 +42,9 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.DEFAULT_DATABASES; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; @@ -57,8 +57,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase { - static Set DEFAULT_DATABASE_FILENAMES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"); - private Path geoipTmpDir; private Path geoIpConfigDir; private ConfigDatabases configDatabases; @@ -74,7 +72,7 @@ public void loadDatabaseReaders() throws IOException { Client client = mock(Client.class); GeoIpCache cache = new GeoIpCache(1000); configDatabases = new ConfigDatabases(geoIpConfigDir, new GeoIpCache(1000)); - copyDatabaseFiles(geoIpConfigDir, configDatabases); + copyDefaultDatabases(geoIpConfigDir, configDatabases); geoipTmpDir = createTempDir(); clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); @@ -181,7 +179,7 @@ public void testBuildDbFile() throws Exception { assertFalse(processor.isIgnoreMissing()); } - public void testBuildWithCountryDbAndAsnFields() throws Exception { + public void testBuildWithCountryDbAndAsnFields() { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -201,7 +199,7 @@ public void testBuildWithCountryDbAndAsnFields() throws Exception { ); } - public void testBuildWithAsnDbAndCityFields() throws Exception { + public void testBuildWithAsnDbAndCityFields() { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -218,10 +216,7 @@ public void testBuildWithAsnDbAndCityFields() throws Exception { } public void testBuildNonExistingDbFile() throws Exception { - Files.copy( - GeoIpProcessorFactoryTests.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), - geoipTmpDir.resolve("GeoLite2-City.mmdb") - ); + copyDatabase("GeoLite2-City-Test.mmdb", geoipTmpDir.resolve("GeoLite2-City.mmdb")); databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City.mmdb")); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); @@ -234,11 +229,11 @@ public void testBuildNonExistingDbFile() throws Exception { public void testBuildBuiltinDatabaseMissing() throws Exception { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); - cleanDatabaseFiles(geoIpConfigDir, configDatabases); + cleanDatabases(geoIpConfigDir, configDatabases); Map config = new HashMap<>(); config.put("field", "_field"); - config.put("database_file", randomFrom(DEFAULT_DATABASE_FILENAMES)); + config.put("database_file", randomFrom(DEFAULT_DATABASES)); Processor processor = factory.create(null, null, null, config); assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class)); } @@ -267,7 +262,7 @@ public void testBuildFields() throws Exception { assertFalse(processor.isIgnoreMissing()); } - public void testBuildIllegalFieldOption() throws Exception { + public void testBuildIllegalFieldOption() { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); Map config1 = new HashMap<>(); @@ -324,14 +319,13 @@ public void testBuildNullDatabase() throws Exception { assertThat(e.getMessage(), equalTo("[database_file] Unsupported database type [null] for file [GeoLite2-City.mmdb]")); } - @SuppressWarnings("HiddenField") public void testLazyLoading() throws Exception { final Path configDir = createTempDir(); final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); Files.createDirectories(geoIpConfigDir); GeoIpCache cache = new GeoIpCache(1000); ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache); - copyDatabaseFiles(geoIpConfigDir, configDatabases); + copyDefaultDatabases(geoIpConfigDir, configDatabases); // Loading another database reader instances, because otherwise we can't test lazy loading as the // database readers used at class level are reused between tests. (we want to keep that otherwise running this @@ -358,7 +352,7 @@ public void testLazyLoading() throws Exception { config.put("database_file", "GeoLite2-City.mmdb"); final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config); - // these are lazy loaded until first use so we expect null here + // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-City.mmdb").databaseReader.get()); city.execute(document); // the first ingest should trigger a database load @@ -369,7 +363,7 @@ public void testLazyLoading() throws Exception { config.put("database_file", "GeoLite2-Country.mmdb"); final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config); - // these are lazy loaded until first use so we expect null here + // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-Country.mmdb").databaseReader.get()); country.execute(document); // the first ingest should trigger a database load @@ -380,22 +374,21 @@ public void testLazyLoading() throws Exception { config.put("database_file", "GeoLite2-ASN.mmdb"); final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config); - // these are lazy loaded until first use so we expect null here + // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get()); asn.execute(document); // the first ingest should trigger a database load assertNotNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get()); } - @SuppressWarnings("HiddenField") public void testLoadingCustomDatabase() throws IOException { final Path configDir = createTempDir(); final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); Files.createDirectories(geoIpConfigDir); ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, new GeoIpCache(1000)); - copyDatabaseFiles(geoIpConfigDir, configDatabases); + copyDefaultDatabases(geoIpConfigDir, configDatabases); // fake the GeoIP2-City database - copyDatabaseFile(geoIpConfigDir, "GeoLite2-City.mmdb"); + copyDatabase("GeoLite2-City.mmdb", geoIpConfigDir); Files.move(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), geoIpConfigDir.resolve("GeoIP2-City.mmdb")); /* @@ -428,7 +421,7 @@ public void testLoadingCustomDatabase() throws IOException { config.put("database_file", "GeoIP2-City.mmdb"); final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config); - // these are lazy loaded until first use so we expect null here + // these are lazy loaded until first use, so we expect null here assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb").databaseReader.get()); city.execute(document); // the first ingest should trigger a database load @@ -490,7 +483,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { assertThat(geoData.get("city_name"), equalTo("Tumba")); } { - copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb"); + copyDatabase("GeoLite2-City-Test.mmdb", geoipTmpDir); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb")); processor.execute(ingestDocument); @@ -498,7 +491,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { assertThat(geoData.get("city_name"), equalTo("Linköping")); } { - // No databases are available, so assume that databases still need to be downloaded and therefor not fail: + // No databases are available, so assume that databases still need to be downloaded and therefore not fail: IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); databaseNodeService.removeStaleEntries(List.of("GeoLite2-City.mmdb")); configDatabases.updateDatabase(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), false); @@ -507,7 +500,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { assertThat(geoData, nullValue()); } { - // There are database available, but not the right one, so tag: + // There are databases available, but not the right one, so tag: databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb")); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); processor.execute(ingestDocument); @@ -517,7 +510,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { public void testDatabaseNotReadyYet() throws Exception { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); - cleanDatabaseFiles(geoIpConfigDir, configDatabases); + cleanDatabases(geoIpConfigDir, configDatabases); { Map config = new HashMap<>(); @@ -542,7 +535,7 @@ public void testDatabaseNotReadyYet() throws Exception { ); } - copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb"); + copyDatabase("GeoLite2-City-Test.mmdb", geoipTmpDir); databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb")); { @@ -562,25 +555,9 @@ public void testDatabaseNotReadyYet() throws Exception { } } - private static void copyDatabaseFile(final Path path, final String databaseFilename) throws IOException { - Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/" + databaseFilename)), - path.resolve(databaseFilename), - StandardCopyOption.REPLACE_EXISTING - ); - } - - static void copyDatabaseFiles(final Path path, ConfigDatabases configDatabases) throws IOException { - for (final String databaseFilename : DEFAULT_DATABASE_FILENAMES) { - copyDatabaseFile(path, databaseFilename); - configDatabases.updateDatabase(path.resolve(databaseFilename), true); + private static void cleanDatabases(final Path directory, ConfigDatabases configDatabases) { + for (final String database : DEFAULT_DATABASES) { + configDatabases.updateDatabase(directory.resolve(database), false); } } - - static void cleanDatabaseFiles(final Path path, ConfigDatabases configDatabases) throws IOException { - for (final String databaseFilename : DEFAULT_DATABASE_FILENAMES) { - configDatabases.updateDatabase(path.resolve(databaseFilename), false); - } - } - } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 87d1881a9e743..762818a7c65db 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -134,7 +134,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { assertIngestDocument(originalIngestDocument, ingestDocument); } - public void testNullWithoutIgnoreMissing() throws Exception { + public void testNullWithoutIgnoreMissing() { GeoIpProcessor processor = new GeoIpProcessor( randomAlphaOfLength(10), null, @@ -156,7 +156,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot extract geoip information.")); } - public void testNonExistentWithoutIgnoreMissing() throws Exception { + public void testNonExistentWithoutIgnoreMissing() { GeoIpProcessor processor = new GeoIpProcessor( randomAlphaOfLength(10), null, @@ -526,7 +526,7 @@ public void testAddressIsNotInTheDatabase() throws Exception { /** * Don't silently do DNS lookups or anything trappy on bogus data */ - public void testInvalid() throws Exception { + public void testInvalid() { GeoIpProcessor processor = new GeoIpProcessor( randomAlphaOfLength(10), null, @@ -803,7 +803,7 @@ long databaseFileSize() throws IOException { } @Override - InputStream databaseInputStream() throws IOException { + InputStream databaseInputStream() { return databaseInputStreamSupplier.get(); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java new file mode 100644 index 0000000000000..a3d72aca2295c --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.core.SuppressForbidden; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +public final class GeoIpTestUtils { + + private GeoIpTestUtils() { + // utility class + } + + public static final Set DEFAULT_DATABASES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"); + + @SuppressForbidden(reason = "uses java.io.File") + private static boolean isDirectory(final Path path) { + return path.toFile().isDirectory(); + } + + public static void copyDatabase(final String databaseName, final Path destination) { + try (InputStream is = GeoIpTestUtils.class.getResourceAsStream("/" + databaseName)) { + if (is == null) { + throw new FileNotFoundException("Resource [" + databaseName + "] not found in classpath"); + } + + Files.copy(is, isDirectory(destination) ? destination.resolve(databaseName) : destination, REPLACE_EXISTING); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void copyDefaultDatabases(final Path directory) { + for (final String database : DEFAULT_DATABASES) { + copyDatabase(database, directory); + } + } + + public static void copyDefaultDatabases(final Path directory, ConfigDatabases configDatabases) { + for (final String database : DEFAULT_DATABASES) { + copyDatabase(database, directory); + configDatabases.updateDatabase(directory.resolve(database), true); + } + } +} diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb index 17fc3715090ae..1b142d0001b9c 100644 Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb index 7ed43d616a85d..04220ff4b6411 100644 Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb index 7bfae78964df0..c49ca3ad48f39 100644 Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb index d21c2a93df7d4..596a96617f241 100644 Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb index 837b725e9c154..16c1acf800260 100644 Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb index d16b0eee4c5e5..a4277d0a55c47 100644 Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb index 0809201619b59..393efe464b610 100644 Binary files a/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb differ