Skip to content

Commit 5f7a151

Browse files
williamrandolphwilliamrandolph
authored
System index auto-creation should not be disabled by user settings (#62984)
* Add System Indices check to AutoCreateIndex By default, Elasticsearch auto-creates indices when a document is submitted to a non-existent index. There is a setting that allows users to disable this behavior. However, this setting should not apply to system indices, so that Elasticsearch modules and plugins are able to use auto-create behavior whether or not it is exposed to users. This commit constructs the AutoCreateIndex object with a reference to the SystemIndices object so that we bypass the check for the user-facing autocreate setting when it's a system index that is being autocreated. We also modify the logic in TransportBulkAction to make sure that if a system index is included in a bulk request, we don't skip the autocreation step.
1 parent ad3b289 commit 5f7a151

File tree

10 files changed

+89
-24
lines changed

10 files changed

+89
-24
lines changed

‎modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,11 @@
3535
import org.elasticsearch.common.bytes.BytesReference;
3636
import org.elasticsearch.common.settings.ClusterSettings;
3737
import org.elasticsearch.common.settings.Settings;
38+
import org.elasticsearch.indices.SystemIndices;
3839
import org.elasticsearch.test.ESTestCase;
3940

41+
import java.util.HashMap;
42+
4043
import static java.util.Collections.emptyMap;
4144
import static org.hamcrest.Matchers.containsString;
4245

@@ -60,7 +63,8 @@ public class ReindexSourceTargetValidationTests extends ESTestCase {
6063
.put(index("source2", "source_multi"), true)).build();
6164
private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver();
6265
private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY,
63-
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER);
66+
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER,
67+
new SystemIndices(new HashMap<>()));
6468

6569
private final BytesReference query = new BytesArray("{ \"foo\" : \"bar\" }");
6670

‎server/src/main/java/org/elasticsearch/action/ActionModule.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@
250250
import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards;
251251
import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
252252
import org.elasticsearch.index.seqno.RetentionLeaseActions;
253+
import org.elasticsearch.indices.SystemIndices;
253254
import org.elasticsearch.indices.breaker.CircuitBreakerService;
254255
import org.elasticsearch.indices.store.TransportNodesListShardStoreMetadata;
255256
import org.elasticsearch.persistent.CompletionPersistentTaskAction;
@@ -426,7 +427,7 @@ public class ActionModule extends AbstractModule {
426427
public ActionModule(Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
427428
IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
428429
ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
429-
CircuitBreakerService circuitBreakerService, UsageService usageService) {
430+
CircuitBreakerService circuitBreakerService, UsageService usageService, SystemIndices systemIndices) {
430431
this.settings = settings;
431432
this.indexNameExpressionResolver = indexNameExpressionResolver;
432433
this.indexScopedSettings = indexScopedSettings;
@@ -436,7 +437,7 @@ public ActionModule(Settings settings, IndexNameExpressionResolver indexNameExpr
436437
this.threadPool = threadPool;
437438
actions = setupActions(actionPlugins);
438439
actionFilters = setupActionFilters(actionPlugins);
439-
autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
440+
autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, systemIndices);
440441
destructiveOperations = new DestructiveOperations(settings, clusterSettings);
441442
Set<RestHeaderDefinition> headers = Stream.concat(
442443
actionPlugins.stream().flatMap(p -> p.getRestHeaders().stream()),

‎server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,9 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, String exec
224224
return;
225225
}
226226

227-
if (needToCheck()) {
227+
final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
228+
229+
if (includesSystem || needToCheck()) {
228230
// Attempt to create all the indices that we're going to need during the bulk before we start.
229231
// Step 1: collect all the indices in the request
230232
final Map<String, Boolean> indices = bulkRequest.requests.stream()
@@ -346,15 +348,20 @@ static void prohibitCustomRoutingOnDataStream(DocWriteRequest<?> writeRequest, M
346348
}
347349

348350
boolean isOnlySystem(BulkRequest request, SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices) {
349-
final boolean onlySystem = request.getIndices().stream().allMatch(indexName -> {
350-
final IndexAbstraction abstraction = indicesLookup.get(indexName);
351-
if (abstraction != null) {
352-
return abstraction.isSystem();
353-
} else {
354-
return systemIndices.isSystemIndex(indexName);
355-
}
356-
});
357-
return onlySystem;
351+
return request.getIndices().stream().allMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName));
352+
}
353+
354+
boolean includesSystem(BulkRequest request, SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices) {
355+
return request.getIndices().stream().anyMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName));
356+
}
357+
358+
private boolean isSystemIndex(SortedMap<String, IndexAbstraction> indicesLookup, SystemIndices systemIndices, String indexName) {
359+
final IndexAbstraction abstraction = indicesLookup.get(indexName);
360+
if (abstraction != null) {
361+
return abstraction.isSystem();
362+
} else {
363+
return systemIndices.isSystemIndex(indexName);
364+
}
358365
}
359366

360367
boolean needToCheck() {

‎server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.elasticsearch.common.settings.Setting.Property;
3131
import org.elasticsearch.common.settings.Settings;
3232
import org.elasticsearch.index.IndexNotFoundException;
33+
import org.elasticsearch.indices.SystemIndices;
3334

3435
import java.util.ArrayList;
3536
import java.util.List;
@@ -44,10 +45,15 @@ public final class AutoCreateIndex {
4445
new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic);
4546

4647
private final IndexNameExpressionResolver resolver;
48+
private final SystemIndices systemIndices;
4749
private volatile AutoCreate autoCreate;
4850

49-
public AutoCreateIndex(Settings settings, ClusterSettings clusterSettings, IndexNameExpressionResolver resolver) {
51+
public AutoCreateIndex(Settings settings,
52+
ClusterSettings clusterSettings,
53+
IndexNameExpressionResolver resolver,
54+
SystemIndices systemIndices) {
5055
this.resolver = resolver;
56+
this.systemIndices = systemIndices;
5157
this.autoCreate = AUTO_CREATE_INDEX_SETTING.get(settings);
5258
clusterSettings.addSettingsUpdateConsumer(AUTO_CREATE_INDEX_SETTING, this::setAutoCreate);
5359
}
@@ -67,6 +73,12 @@ public boolean shouldAutoCreate(String index, ClusterState state) {
6773
if (resolver.hasIndexAbstraction(index, state)) {
6874
return false;
6975
}
76+
77+
// Always auto-create system indexes
78+
if (systemIndices.isSystemIndex(index)) {
79+
return true;
80+
}
81+
7082
// One volatile read, so that all checks are done against the same instance:
7183
final AutoCreate autoCreate = this.autoCreate;
7284
if (autoCreate.autoCreateIndex == false) {

‎server/src/main/java/org/elasticsearch/node/Node.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -522,7 +522,7 @@ protected Node(final Environment initialEnvironment,
522522

523523
ActionModule actionModule = new ActionModule(settings, clusterModule.getIndexNameExpressionResolver(),
524524
settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
525-
threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService);
525+
threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService, systemIndices);
526526
modules.add(actionModule);
527527

528528
final RestController restController = actionModule.getRestController();

‎server/src/test/java/org/elasticsearch/action/ActionModuleTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() {
109109
UsageService usageService = new UsageService();
110110
ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
111111
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
112-
null, usageService);
112+
null, usageService, null);
113113
actionModule.initRestHandlers(null);
114114
// At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
115115
Exception e = expectThrows(IllegalArgumentException.class, () ->
@@ -148,7 +148,7 @@ public String getName() {
148148
UsageService usageService = new UsageService();
149149
ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
150150
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
151-
singletonList(dupsMainAction), null, null, usageService);
151+
singletonList(dupsMainAction), null, null, usageService, null);
152152
Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
153153
assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/] for method: GET"));
154154
} finally {
@@ -182,7 +182,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
182182
UsageService usageService = new UsageService();
183183
ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
184184
settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
185-
singletonList(registersFakeHandler), null, null, usageService);
185+
singletonList(registersFakeHandler), null, null, usageService, null);
186186
actionModule.initRestHandlers(null);
187187
// At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
188188
Exception e = expectThrows(IllegalArgumentException.class, () ->

‎server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,9 @@ class TestTransportBulkAction extends TransportBulkAction {
145145
null, new ActionFilters(Collections.emptySet()), null,
146146
new AutoCreateIndex(
147147
SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
148-
new IndexNameExpressionResolver()
149-
), new IndexingPressure(SETTINGS), new SystemIndices(Map.of())
148+
new IndexNameExpressionResolver(),
149+
new SystemIndices(Map.of())),
150+
new IndexingPressure(SETTINGS), new SystemIndices(Map.of())
150151
);
151152
}
152153
@Override

‎server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ class TestTransportBulkAction extends TransportBulkAction {
8585
TestTransportBulkAction() {
8686
super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null,
8787
null, new ActionFilters(Collections.emptySet()), new Resolver(),
88-
new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver()),
88+
new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(), new SystemIndices(Map.of())),
8989
new IndexingPressure(Settings.EMPTY), new SystemIndices(Map.of()));
9090
}
9191

@@ -268,6 +268,27 @@ public void testOnlySystem() {
268268
assertFalse(bulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
269269
}
270270

271+
public void testIncludesSystem() {
272+
SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
273+
Settings settings = Settings.builder().put("index.version.created", Version.CURRENT).build();
274+
indicesLookup.put(".foo",
275+
new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
276+
indicesLookup.put(".bar",
277+
new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
278+
SystemIndices systemIndices = new SystemIndices(Map.of("plugin", List.of(new SystemIndexDescriptor(".test", ""))));
279+
List<String> onlySystem = List.of(".foo", ".bar");
280+
assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
281+
282+
onlySystem = List.of(".foo", ".bar", ".test");
283+
assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
284+
285+
List<String> nonSystem = List.of("foo", "bar");
286+
assertFalse(bulkAction.includesSystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices));
287+
288+
List<String> mixed = List.of(".foo", ".test", "other");
289+
assertTrue(bulkAction.includesSystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
290+
}
291+
271292
private BulkRequest buildBulkRequest(List<String> indices) {
272293
BulkRequest request = new BulkRequest();
273294
for (String index : indices) {

‎server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.elasticsearch.common.settings.ClusterSettings;
3030
import org.elasticsearch.common.settings.Settings;
3131
import org.elasticsearch.index.IndexNotFoundException;
32+
import org.elasticsearch.indices.SystemIndexDescriptor;
33+
import org.elasticsearch.indices.SystemIndices;
3234
import org.elasticsearch.test.ESTestCase;
3335

3436
import java.util.HashMap;
@@ -39,6 +41,8 @@
3941

4042
public class AutoCreateIndexTests extends ESTestCase {
4143

44+
private static final String TEST_SYSTEM_INDEX_NAME = ".test-system-index";
45+
4246
public void testParseFailed() {
4347
try {
4448
Settings settings = Settings.builder().put("action.auto_create_index", ",,,").build();
@@ -89,6 +93,12 @@ public void testAutoCreationDisabled() {
8993
assertEquals("no such index [" + randomIndex + "] and [action.auto_create_index] is [false]", e.getMessage());
9094
}
9195

96+
public void testSystemIndexWithAutoCreationDisabled() {
97+
Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build();
98+
AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
99+
assertThat(autoCreateIndex.shouldAutoCreate(TEST_SYSTEM_INDEX_NAME, buildClusterState()), equalTo(true));
100+
}
101+
92102
public void testAutoCreationEnabled() {
93103
Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), true).build();
94104
AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
@@ -127,6 +137,13 @@ public void testAutoCreationPatternDisabled() {
127137
expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAlphaOfLengthBetween(1, 5));
128138
}
129139

140+
public void testAutoCreationSystemIndexPatternDisabled() {
141+
Settings settings =
142+
Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), "-" + TEST_SYSTEM_INDEX_NAME + "*").build();
143+
AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
144+
assertThat(autoCreateIndex.shouldAutoCreate(TEST_SYSTEM_INDEX_NAME, buildClusterState()), equalTo(true));
145+
}
146+
130147
public void testAutoCreationMultiplePatternsWithWildcards() {
131148
Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(),
132149
randomFrom("+test*,-index*", "test*,-index*")).build();
@@ -177,7 +194,8 @@ public void testUpdate() {
177194

178195
ClusterSettings clusterSettings = new ClusterSettings(settings,
179196
ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
180-
AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, new IndexNameExpressionResolver());
197+
AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, new IndexNameExpressionResolver(),
198+
new SystemIndices(Map.of()));
181199
assertThat(autoCreateIndex.getAutoCreate().isAutoCreateIndex(), equalTo(value));
182200

183201
Settings newSettings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), !value).build();
@@ -201,8 +219,9 @@ private static ClusterState buildClusterState(String... indices) {
201219
}
202220

203221
private AutoCreateIndex newAutoCreateIndex(Settings settings) {
222+
SystemIndices systemIndices = new SystemIndices(Map.of("plugin", List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, ""))));
204223
return new AutoCreateIndex(settings, new ClusterSettings(settings,
205-
ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver());
224+
ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(), systemIndices);
206225
}
207226

208227
private void expectNotMatch(ClusterState clusterState, AutoCreateIndex autoCreateIndex, String index) {

‎server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1578,7 +1578,7 @@ allocationService, new AliasValidator(), shardLimitValidator, environment, index
15781578
new AnalysisModule(environment, Collections.emptyList()).getAnalysisRegistry(),
15791579
Collections.emptyList(), client),
15801580
client, actionFilters, indexNameExpressionResolver,
1581-
new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver),
1581+
new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, new SystemIndices(Map.of())),
15821582
new IndexingPressure(settings),
15831583
new SystemIndices(Map.of())
15841584
));

0 commit comments

Comments
 (0)