Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update geoip fixture files and utility methods #111913

Merged
merged 8 commits into from
Aug 15, 2024
Merged

Update geoip fixture files and utility methods #111913

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
64a4368
Update -Test mmdb files
joegallo Aug 13, 2024
5073f34
Make this private static final
joegallo Aug 13, 2024
58b2307
Add commas to silence IntelliJ
joegallo Aug 13, 2024
ee62e1c
Quiet some warnings from IntelliJ
joegallo Aug 13, 2024
973e929
Remove redundant warning suppression
joegallo Aug 13, 2024
c66c3f6
Fix some typos
joegallo Aug 13, 2024
d90256e
createTempDir does what it says
joegallo Aug 14, 2024
7030349
Rewrite the database copying utilities
joegallo Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 3 additions & 22 deletions ...st-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,14 @@
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.StreamsUtils;
import org.junit.ClassRule;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;

import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;

public abstract class AbstractGeoIpIT extends ESIntegTestCase {
private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false;

Expand All @@ -45,23 +42,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
@Override
protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSettings) {
final Path databasePath = createTempDir();
try {
Files.createDirectories(databasePath);
Files.copy(
new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")),
databasePath.resolve("GeoLite2-City.mmdb")
);
Files.copy(
new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")),
databasePath.resolve("GeoLite2-Country.mmdb")
);
Files.copy(
new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")),
databasePath.resolve("GeoLite2-ASN.mmdb")
);
} catch (final IOException e) {
throw new UncheckedIOException(e);
}
copyDefaultDatabases(databasePath);
return Settings.builder()
.put("ingest.geoip.database_path", databasePath)
.put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false)
Expand Down
8 changes: 2 additions & 6 deletions ...-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import java.util.zip.GZIPInputStream;

import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
import static org.hamcrest.Matchers.anEmptyMap;
Expand Down Expand Up @@ -688,12 +689,7 @@ private void setupDatabasesInConfigDirectory() throws Exception {
.forEach(path -> {
try {
Files.createDirectories(path);
Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-City.mmdb"), path.resolve("GeoLite2-City.mmdb"));
Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), path.resolve("GeoLite2-ASN.mmdb"));
Files.copy(
GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-Country.mmdb"),
path.resolve("GeoLite2-Country.mmdb")
);
copyDefaultDatabases(path);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
Expand Down
23 changes: 9 additions & 14 deletions ...st/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,8 @@
import org.elasticsearch.watcher.ResourceWatcherService;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
Expand All @@ -34,7 +32,8 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;

import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
Expand Down Expand Up @@ -68,8 +67,8 @@ public void test() throws Exception {
when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir, clusterService);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
lazyLoadReaders(databaseNodeService);
Expand Down Expand Up @@ -138,18 +137,14 @@ public void test() throws Exception {
assertThat(previous1.current(), equalTo(-1));
});
} else {
Files.copy(
ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
geoIpTmpDir.resolve("GeoLite2-City.mmdb"),
StandardCopyOption.REPLACE_EXISTING
);
copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
}
DatabaseReaderLazyLoader previous2 = databaseNodeService.get("GeoLite2-City-Test.mmdb");
InputStream source = ConfigDatabases.class.getResourceAsStream(
i % 2 == 0 ? "/GeoIP2-City-Test.mmdb" : "/GeoLite2-City-Test.mmdb"
copyDatabase(
i % 2 == 0 ? "GeoIP2-City-Test.mmdb" : "GeoLite2-City-Test.mmdb",
geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")
);
Files.copy(source, geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), StandardCopyOption.REPLACE_EXISTING);
databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));

DatabaseReaderLazyLoader current1 = databaseNodeService.get("GeoLite2-City.mmdb");
Expand Down Expand Up @@ -194,7 +189,7 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI
throws IOException {
GeoIpCache cache = new GeoIpCache(0);
ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
copyDatabaseFiles(geoIpConfigDir, configDatabases);
copyDefaultDatabases(geoIpConfigDir, configDatabases);
DatabaseNodeService databaseNodeService = new DatabaseNodeService(
geoIpTmpDir,
mock(Client.class),
Expand Down
30 changes: 12 additions & 18 deletions modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,11 @@
import org.junit.After;
import org.junit.Before;

import java.io.IOException;
import java.nio.file.CopyOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;

import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
Expand Down Expand Up @@ -62,8 +61,8 @@ public void testLocalDatabasesEmptyConfig() throws Exception {

public void testDatabasesConfigDir() throws Exception {
Path configDir = createTempDir();
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));

ConfigDatabases configDatabases = new ConfigDatabases(configDir, new GeoIpCache(0));
configDatabases.initialize(resourceWatcherService);
Expand Down Expand Up @@ -92,9 +91,9 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {
assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country"));
}

CopyOption option = StandardCopyOption.REPLACE_EXISTING;
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), option);
copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));

assertBusy(() -> {
assertThat(configDatabases.getConfigDatabases().size(), equalTo(4));
DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-ASN.mmdb");
Expand All @@ -116,7 +115,8 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {

public void testDatabasesUpdateExistingConfDatabase() throws Exception {
Path configDir = createTempDir();
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
copyDatabase("GeoLite2-City.mmdb", configDir);

GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
configDatabases.initialize(resourceWatcherService);
Expand All @@ -131,11 +131,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
assertThat(cache.count(), equalTo(1));
}

Files.copy(
ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
configDir.resolve("GeoLite2-City.mmdb"),
StandardCopyOption.REPLACE_EXISTING
);
copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
assertBusy(() -> {
assertThat(configDatabases.getConfigDatabases().size(), equalTo(1));
assertThat(cache.count(), equalTo(0));
Expand All @@ -154,11 +150,9 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
});
}

private static Path prepareConfigDir() throws IOException {
private static Path prepareConfigDir() {
Path dir = createTempDir();
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), dir.resolve("GeoLite2-ASN.mmdb"));
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), dir.resolve("GeoLite2-City.mmdb"));
Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-Country.mmdb"), dir.resolve("GeoLite2-Country.mmdb"));
copyDefaultDatabases(dir);
return dir;
}

Expand Down
5 changes: 2 additions & 3 deletions ...s/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.TYPE;
import static org.hamcrest.Matchers.empty;
Expand Down Expand Up @@ -117,10 +117,9 @@ public class DatabaseNodeServiceTests extends ESTestCase {
@Before
public void setup() throws IOException {
final Path geoIpConfigDir = createTempDir();
Files.createDirectories(geoIpConfigDir);
GeoIpCache cache = new GeoIpCache(1000);
ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
copyDatabaseFiles(geoIpConfigDir, configDatabases);
copyDefaultDatabases(geoIpConfigDir, configDatabases);

threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName());
Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build();
Expand Down
Loading