-
Notifications
You must be signed in to change notification settings - Fork 24.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
It turns out that there is a fairly simple recipe for reindexing to a `time_series` index: 1. If you are reindexing from a time series index to a time series index and *not* changing the `@timestamp` or dimensions it "just works"(TM). 2. If you are reindexing from a standard index with a standard random `_id` you should clear it on reindex. 3. If you are reindexing from tsdb index to a tsdb index and modifying a dimension or `@timestamp` then you should clear the `_id`. This is not pleasant to have to remember. But it doesn't crash! * TSDB: Initial reindex fix This teaches reindex the smallest thing that it needs to know about tsdb: `_id` is automatically generated. Armed with that knowledge reindex now doesn't attempt to copy the `_id` when writing to a tsdb index. Important: If the index doesn't yet exist it will *assume* that the index will be created in `standard` mode. We can detect what mode it *should* be created with in a follow up change.
- Loading branch information
Showing
8 changed files
with
251 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.reindex; | ||
|
|
||
| import org.elasticsearch.Version; | ||
| import org.elasticsearch.cluster.ClusterState; | ||
| import org.elasticsearch.cluster.metadata.IndexMetadata; | ||
| import org.elasticsearch.cluster.metadata.Metadata; | ||
| import org.elasticsearch.common.bytes.BytesArray; | ||
| import org.elasticsearch.common.settings.Settings; | ||
| import org.elasticsearch.index.IndexMode; | ||
| import org.elasticsearch.index.IndexSettings; | ||
| import org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollActionTestCase; | ||
| import org.elasticsearch.index.reindex.BulkByScrollResponse; | ||
| import org.elasticsearch.index.reindex.ReindexRequest; | ||
| import org.elasticsearch.index.reindex.ScrollableHitSource; | ||
| import org.elasticsearch.xcontent.XContentType; | ||
|
|
||
| import static org.hamcrest.Matchers.equalTo; | ||
| import static org.hamcrest.Matchers.nullValue; | ||
|
|
||
| /** | ||
| * Reindex | ||
| */ | ||
| public class ReindexIdTests extends AbstractAsyncBulkByScrollActionTestCase<ReindexRequest, BulkByScrollResponse> { | ||
| public void testEmptyStateCopiesId() throws Exception { | ||
| assertThat(action(ClusterState.EMPTY_STATE).buildRequest(doc()).getId(), equalTo(doc().getId())); | ||
| } | ||
|
|
||
| public void testStandardIndexCopiesId() throws Exception { | ||
| Settings.Builder settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD); | ||
| assertThat(action(stateWithIndex(settings)).buildRequest(doc()).getId(), equalTo(doc().getId())); | ||
| } | ||
|
|
||
| public void testTsdbIndexClearsId() throws Exception { | ||
| Settings.Builder settings = Settings.builder() | ||
| .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) | ||
| .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); | ||
| assertThat(action(stateWithIndex(settings)).buildRequest(doc()).getId(), nullValue()); | ||
| } | ||
|
|
||
| private ClusterState stateWithIndex(Settings.Builder settings) { | ||
| IndexMetadata.Builder meta = IndexMetadata.builder(request().getDestination().index()) | ||
| .settings(settings.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) | ||
| .numberOfReplicas(0) | ||
| .numberOfShards(1); | ||
| return ClusterState.builder(ClusterState.EMPTY_STATE).metadata(Metadata.builder(Metadata.EMPTY_METADATA).put(meta)).build(); | ||
| } | ||
|
|
||
| private ScrollableHitSource.BasicHit doc() { | ||
| return new ScrollableHitSource.BasicHit("index", "id", -1).setSource(new BytesArray("{}"), XContentType.JSON); | ||
| } | ||
|
|
||
| @Override | ||
| protected ReindexRequest request() { | ||
| return new ReindexRequest().setDestIndex("dest_index"); | ||
| } | ||
|
|
||
| private Reindexer.AsyncIndexBySearchAction action(ClusterState state) { | ||
| return new Reindexer.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, state, null, request(), listener()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.