Skip to content

Commit 20c4e45

Browse files
tlrxtlrx
authored
Move testSnapshotWithLargeSegmentFiles to ESMockAPIBasedRepositoryIntegTestCase (#46802)
This commit moves the common test testSnapshotWithLargeSegmentFiles to the ESMockAPIBasedRepositoryIntegTestCase base class.
1 parent e0b19a8 commit 20c4e45

File tree

6 files changed

+51
-136
lines changed

6 files changed

+51
-136
lines changed

‎plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java‎

Lines changed: 1 addition & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
import com.microsoft.azure.storage.blob.BlobRequestOptions;
2525
import com.sun.net.httpserver.HttpExchange;
2626
import com.sun.net.httpserver.HttpHandler;
27-
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
28-
import org.elasticsearch.cluster.metadata.IndexMetaData;
2927
import org.elasticsearch.common.SuppressForbidden;
3028
import org.elasticsearch.common.bytes.BytesArray;
3129
import org.elasticsearch.common.bytes.BytesReference;
@@ -34,13 +32,10 @@
3432
import org.elasticsearch.common.settings.MockSecureSettings;
3533
import org.elasticsearch.common.settings.Settings;
3634
import org.elasticsearch.common.unit.ByteSizeUnit;
37-
import org.elasticsearch.common.unit.TimeValue;
38-
import org.elasticsearch.index.IndexSettings;
3935
import org.elasticsearch.plugins.Plugin;
4036
import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase;
4137
import org.elasticsearch.rest.RestStatus;
4238
import org.elasticsearch.rest.RestUtils;
43-
import org.elasticsearch.test.BackgroundIndexer;
4439

4540
import java.io.ByteArrayOutputStream;
4641
import java.io.IOException;
@@ -58,10 +53,6 @@
5853
import java.util.regex.Pattern;
5954
import java.util.stream.Collectors;
6055

61-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
62-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
63-
import static org.hamcrest.Matchers.equalTo;
64-
6556
@SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint")
6657
public class AzureBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase {
6758

@@ -73,6 +64,7 @@ protected String repositoryType() {
7364
@Override
7465
protected Settings repositorySettings() {
7566
return Settings.builder()
67+
.put(super.repositorySettings())
7668
.put(AzureRepository.Repository.CONTAINER_SETTING.getKey(), "container")
7769
.put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test")
7870
.build();
@@ -108,41 +100,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
108100
.build();
109101
}
110102

111-
/**
112-
* Test the snapshot and restore of an index which has large segments files.
113-
*/
114-
public void testSnapshotWithLargeSegmentFiles() throws Exception {
115-
final String repository = createRepository(randomName());
116-
final String index = "index-no-merges";
117-
createIndex(index, Settings.builder()
118-
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
119-
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
120-
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE)
121-
.build());
122-
123-
// the number of documents here dictates the size of the single segment
124-
// we want a large segment (1Mb+) so that Azure SDK client executes Put Block API calls
125-
// the size of each uploaded block is defined by Constants.DEFAULT_STREAM_WRITE_IN_BYTES (~4Mb)
126-
final long nbDocs = randomLongBetween(10_000L, 20_000L);
127-
try (BackgroundIndexer indexer = new BackgroundIndexer(index, "_doc", client(), (int) nbDocs)) {
128-
awaitBusy(() -> indexer.totalIndexedDocs() >= nbDocs);
129-
}
130-
131-
flushAndRefresh(index);
132-
ForceMergeResponse forceMerge = client().admin().indices().prepareForceMerge(index).setFlush(true).setMaxNumSegments(1).get();
133-
assertThat(forceMerge.getSuccessfulShards(), equalTo(1));
134-
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
135-
136-
assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repository, "snapshot")
137-
.setWaitForCompletion(true).setIndices(index));
138-
139-
assertAcked(client().admin().indices().prepareDelete(index));
140-
141-
assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repository, "snapshot").setWaitForCompletion(true));
142-
ensureGreen(index);
143-
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
144-
}
145-
146103
/**
147104
* AzureRepositoryPlugin that allows to set low values for the Azure's client retry policy
148105
* and for BlobRequestOptions#getSingleBlobPutThresholdInBytes().

‎plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java‎

Lines changed: 0 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,12 @@
2020
package org.elasticsearch.repositories.gcs;
2121

2222
import com.google.api.gax.retrying.RetrySettings;
23-
import com.google.cloud.BaseWriteChannel;
2423
import com.google.cloud.http.HttpTransportOptions;
2524
import com.google.cloud.storage.StorageOptions;
2625
import com.sun.net.httpserver.HttpExchange;
2726
import com.sun.net.httpserver.HttpHandler;
2827
import org.apache.http.HttpStatus;
2928
import org.apache.lucene.util.ArrayUtil;
30-
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
31-
import org.elasticsearch.cluster.metadata.IndexMetaData;
3229
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
3330
import org.elasticsearch.common.Strings;
3431
import org.elasticsearch.common.SuppressForbidden;
@@ -49,7 +46,6 @@
4946
import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase;
5047
import org.elasticsearch.rest.RestStatus;
5148
import org.elasticsearch.rest.RestUtils;
52-
import org.elasticsearch.test.BackgroundIndexer;
5349
import org.elasticsearch.threadpool.ThreadPool;
5450
import org.threeten.bp.Duration;
5551

@@ -81,9 +77,6 @@
8177
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.TOKEN_URI_SETTING;
8278
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.BUCKET;
8379
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.CLIENT_NAME;
84-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
85-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
86-
import static org.hamcrest.Matchers.equalTo;
8780

8881
@SuppressForbidden(reason = "this test uses a HttpServer to emulate a Google Cloud Storage endpoint")
8982
public class GoogleCloudStorageBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase {
@@ -177,44 +170,6 @@ public void testChunkSize() {
177170
assertEquals("failed to parse value [101mb] for setting [chunk_size], must be <= [100mb]", e.getMessage());
178171
}
179172

180-
/**
181-
* Test the snapshot and restore of an index which has large segments files (2Mb+).
182-
*
183-
* The value of 2Mb is chosen according to the default chunk size configured in Google SDK client
184-
* (see {@link BaseWriteChannel} chunk size).
185-
*/
186-
public void testSnapshotWithLargeSegmentFiles() throws Exception {
187-
final String repository = createRepository("repository", Settings.builder()
188-
.put(BUCKET.getKey(), "bucket")
189-
.put(CLIENT_NAME.getKey(), "test")
190-
.build());
191-
192-
final String index = "index-no-merges";
193-
createIndex(index, Settings.builder()
194-
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
195-
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
196-
.build());
197-
198-
final int nbDocs = 10_000;
199-
try (BackgroundIndexer indexer = new BackgroundIndexer(index, "_doc", client(), nbDocs)) {
200-
waitForDocs(nbDocs, indexer);
201-
}
202-
203-
flushAndRefresh(index);
204-
ForceMergeResponse forceMerge = client().admin().indices().prepareForceMerge(index).setMaxNumSegments(1).get();
205-
assertThat(forceMerge.getSuccessfulShards(), equalTo(1));
206-
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
207-
208-
assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repository, "snapshot")
209-
.setWaitForCompletion(true).setIndices(index));
210-
211-
assertAcked(client().admin().indices().prepareDelete(index));
212-
213-
assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repository, "snapshot").setWaitForCompletion(true));
214-
ensureGreen(index);
215-
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
216-
}
217-
218173
public static class TestGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin {
219174

220175
public TestGoogleCloudStoragePlugin(Settings settings) {

‎plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java‎

Lines changed: 1 addition & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@
2323
import com.amazonaws.util.Base16;
2424
import com.sun.net.httpserver.HttpExchange;
2525
import com.sun.net.httpserver.HttpHandler;
26-
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
27-
import org.elasticsearch.cluster.metadata.IndexMetaData;
2826
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
2927
import org.elasticsearch.common.SuppressForbidden;
3028
import org.elasticsearch.common.UUIDs;
@@ -45,7 +43,6 @@
4543
import org.elasticsearch.rest.RestStatus;
4644
import org.elasticsearch.rest.RestUtils;
4745
import org.elasticsearch.snapshots.mockstore.BlobStoreWrapper;
48-
import org.elasticsearch.test.BackgroundIndexer;
4946
import org.elasticsearch.threadpool.ThreadPool;
5047

5148
import java.io.ByteArrayOutputStream;
@@ -63,9 +60,6 @@
6360
import java.util.concurrent.ConcurrentMap;
6461

6562
import static java.nio.charset.StandardCharsets.UTF_8;
66-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
67-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
68-
import static org.hamcrest.Matchers.equalTo;
6963
import static org.hamcrest.Matchers.nullValue;
7064

7165
@SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint")
@@ -79,6 +73,7 @@ protected String repositoryType() {
7973
@Override
8074
protected Settings repositorySettings() {
8175
return Settings.builder()
76+
.put(super.repositorySettings())
8277
.put(S3Repository.BUCKET_SETTING.getKey(), "bucket")
8378
.put(S3Repository.CLIENT_NAME.getKey(), "test")
8479
.build();
@@ -116,34 +111,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
116111
.build();
117112
}
118113

119-
public void testSnapshotWithLargeSegmentFiles() throws Exception {
120-
final String repository = createRepository(randomName());
121-
final String index = "index-no-merges";
122-
createIndex(index, Settings.builder()
123-
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
124-
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
125-
.build());
126-
127-
final long nbDocs = randomLongBetween(10_000L, 20_000L);
128-
try (BackgroundIndexer indexer = new BackgroundIndexer(index, "_doc", client(), (int) nbDocs)) {
129-
waitForDocs(nbDocs, indexer);
130-
}
131-
132-
flushAndRefresh(index);
133-
ForceMergeResponse forceMerge = client().admin().indices().prepareForceMerge(index).setFlush(true).setMaxNumSegments(1).get();
134-
assertThat(forceMerge.getSuccessfulShards(), equalTo(1));
135-
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
136-
137-
assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repository, "snapshot")
138-
.setWaitForCompletion(true).setIndices(index));
139-
140-
assertAcked(client().admin().indices().prepareDelete(index));
141-
142-
assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repository, "snapshot").setWaitForCompletion(true));
143-
ensureGreen(index);
144-
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
145-
}
146-
147114
/**
148115
* S3RepositoryPlugin that allows to disable chunked encoding and to set a low threshold between single upload and multipart upload.
149116
*/

‎server/src/test/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.ElasticsearchException;
2222
import org.elasticsearch.common.settings.Settings;
2323
import org.elasticsearch.common.unit.ByteSizeUnit;
24+
import org.elasticsearch.common.unit.ByteSizeValue;
2425
import org.elasticsearch.core.internal.io.IOUtils;
2526
import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase;
2627

@@ -44,10 +45,14 @@ protected String repositoryType() {
4445

4546
@Override
4647
protected Settings repositorySettings() {
47-
return Settings.builder()
48-
.put(super.repositorySettings())
49-
.put("location", randomRepoPath())
50-
.build();
48+
final Settings.Builder settings = Settings.builder();
49+
settings.put(super.repositorySettings());
50+
settings.put("location", randomRepoPath());
51+
if (randomBoolean()) {
52+
long size = 1 << randomInt(10);
53+
settings.put("chunk_size", new ByteSizeValue(size, ByteSizeUnit.KB));
54+
}
55+
return settings.build();
5156
}
5257

5358
public void testMissingDirectoriesNotCreatedInReadonlyRepository() throws IOException, ExecutionException, InterruptedException {

‎test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,6 @@
2828
import org.elasticsearch.common.blobstore.BlobContainer;
2929
import org.elasticsearch.common.blobstore.BlobStore;
3030
import org.elasticsearch.common.settings.Settings;
31-
import org.elasticsearch.common.unit.ByteSizeUnit;
32-
import org.elasticsearch.common.unit.ByteSizeValue;
3331
import org.elasticsearch.repositories.IndexId;
3432
import org.elasticsearch.repositories.RepositoriesService;
3533
import org.elasticsearch.repositories.RepositoryData;
@@ -63,13 +61,7 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
6361
protected abstract String repositoryType();
6462

6563
protected Settings repositorySettings() {
66-
final Settings.Builder settings = Settings.builder();
67-
settings.put("compress", randomBoolean());
68-
if (randomBoolean()) {
69-
long size = 1 << randomInt(10);
70-
settings.put("chunk_size", new ByteSizeValue(size, ByteSizeUnit.KB));
71-
}
72-
return settings.build();
64+
return Settings.builder().put("compress", randomBoolean()).build();
7365
}
7466

7567
protected final String createRepository(final String name) {

‎test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,15 @@
2222
import com.sun.net.httpserver.HttpHandler;
2323
import com.sun.net.httpserver.HttpServer;
2424
import org.apache.http.HttpStatus;
25+
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
26+
import org.elasticsearch.cluster.metadata.IndexMetaData;
2527
import org.elasticsearch.common.Strings;
2628
import org.elasticsearch.common.SuppressForbidden;
2729
import org.elasticsearch.common.io.Streams;
2830
import org.elasticsearch.common.network.InetAddresses;
31+
import org.elasticsearch.common.settings.Settings;
2932
import org.elasticsearch.mocksocket.MockHttpServer;
33+
import org.elasticsearch.test.BackgroundIndexer;
3034
import org.junit.After;
3135
import org.junit.AfterClass;
3236
import org.junit.Before;
@@ -39,6 +43,10 @@
3943
import java.util.concurrent.ConcurrentHashMap;
4044
import java.util.concurrent.atomic.AtomicInteger;
4145

46+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
47+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
48+
import static org.hamcrest.Matchers.equalTo;
49+
4250
/**
4351
* Integration tests for {@link BlobStoreRepository} implementations rely on mock APIs that emulate cloud-based services.
4452
*/
@@ -83,6 +91,37 @@ public void tearDownHttpServer() {
8391

8492
protected abstract HttpHandler createErroneousHttpHandler(HttpHandler delegate);
8593

94+
/**
95+
* Test the snapshot and restore of an index which has large segments files.
96+
*/
97+
public final void testSnapshotWithLargeSegmentFiles() throws Exception {
98+
final String repository = createRepository(randomName());
99+
final String index = "index-no-merges";
100+
createIndex(index, Settings.builder()
101+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
102+
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
103+
.build());
104+
105+
final long nbDocs = randomLongBetween(10_000L, 20_000L);
106+
try (BackgroundIndexer indexer = new BackgroundIndexer(index, "_doc", client(), (int) nbDocs)) {
107+
waitForDocs(nbDocs, indexer);
108+
}
109+
110+
flushAndRefresh(index);
111+
ForceMergeResponse forceMerge = client().admin().indices().prepareForceMerge(index).setFlush(true).setMaxNumSegments(1).get();
112+
assertThat(forceMerge.getSuccessfulShards(), equalTo(1));
113+
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
114+
115+
assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repository, "snapshot")
116+
.setWaitForCompletion(true).setIndices(index));
117+
118+
assertAcked(client().admin().indices().prepareDelete(index));
119+
120+
assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repository, "snapshot").setWaitForCompletion(true));
121+
ensureGreen(index);
122+
assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
123+
}
124+
86125
protected static String httpServerUrl() {
87126
InetSocketAddress address = httpServer.getAddress();
88127
return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort();

0 commit comments

Comments
 (0)