-
Notifications
You must be signed in to change notification settings - Fork 24.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add the ability to require an ingest pipeline (#46847)
This commit adds the ability to require an ingest pipeline on an index. Today we can have a default pipeline, but that could be overridden by a request pipeline parameter. This commit introduces a new index setting index.required_pipeline that acts similarly to index.default_pipeline, except that it can not be overridden by a request pipeline parameter. Additionally, a default pipeline and a request pipeline can not both be set. The required pipeline can be set to _none to ensure that no pipeline ever runs for index requests on that index.
- 7.10
- 7.11
- 7.12
- 7.13
- 7.14
- 7.15
- 7.16
- 7.17
- 7.17-ci-pipelines-add-ubuntu-2404
- 7.5
- 7.6
- 7.7
- 7.8
- 7.9
- AndyHunt66-patch-1
- Leaf-Lin-patch-2
- PhaedrusTheGreek-patch-2
- (#54905)
- TheRiffRafi-patch-1
- add-linux-aarch64-to-bwc-setup-plugin-7.x
- albertzaharovits-doc-dls-fls-remote-index-permitted
- backport/7.17/pr-116172
- backport/undo_transient_settings
- build-bwc-without-es-runtime-7x
- buildkite-migration-717
- crisdarocha-patch-1
- fix-indenting-code-block
- freakingid-ilm-action-forcemerge
- geekpete-patch-1
- ignore-older-version-bwc-tests-with-aarch64-7.x
- jeanfabrice-docpatch-1
- kilfoyle-patch-1
- kilfoyle-remove-frozen-tier-warning
- nathandh22-patch-1
- (#88967)
- perrinal-patch-1
- (#55631)
- ppf2-aggregations-ccs-leak-7.15.0
- ppf2-improve-slack-setup
- release-notes-7-17-20
- richiejarvis-patch-2
- robin13-patch-1
- (ivansun1010/elasticsearch#3)
- rseldner-patch-1
- rseldner-patch-1-1
- rseldner-patch-1-2
- toby-sutor-patch-2
1 parent
251dbd8
commit bd77626
Showing
9 changed files
with
513 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
175 changes: 175 additions & 0 deletions
175
modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/240_required_pipeline.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,175 @@ | ||
--- | ||
teardown: | ||
- do: | ||
ingest.delete_pipeline: | ||
id: "my_pipeline" | ||
ignore: 404 | ||
|
||
--- | ||
"Test index with required pipeline": | ||
- do: | ||
ingest.put_pipeline: | ||
id: "my_pipeline" | ||
body: > | ||
{ | ||
"description": "_description", | ||
"processors": [ | ||
{ | ||
"bytes" : { | ||
"field" : "bytes_source_field", | ||
"target_field" : "bytes_target_field" | ||
} | ||
} | ||
] | ||
} | ||
- match: { acknowledged: true } | ||
# required pipeline via index | ||
- do: | ||
indices.create: | ||
index: test | ||
body: | ||
settings: | ||
index: | ||
required_pipeline: "my_pipeline" | ||
aliases: | ||
test_alias: {} | ||
|
||
- do: | ||
index: | ||
index: test | ||
id: 1 | ||
body: {bytes_source_field: "1kb"} | ||
|
||
- do: | ||
get: | ||
index: test | ||
id: 1 | ||
- match: { _source.bytes_source_field: "1kb" } | ||
- match: { _source.bytes_target_field: 1024 } | ||
# required pipeline via alias | ||
- do: | ||
index: | ||
index: test_alias | ||
id: 2 | ||
body: {bytes_source_field: "1kb"} | ||
|
||
- do: | ||
get: | ||
index: test | ||
id: 2 | ||
- match: { _source.bytes_source_field: "1kb" } | ||
- match: { _source.bytes_target_field: 1024 } | ||
# required pipeline via upsert | ||
- do: | ||
update: | ||
index: test | ||
id: 3 | ||
body: | ||
script: | ||
source: "ctx._source.ran_script = true" | ||
lang: "painless" | ||
upsert: { "bytes_source_field":"1kb" } | ||
- do: | ||
get: | ||
index: test | ||
id: 3 | ||
- match: { _source.bytes_source_field: "1kb" } | ||
- match: { _source.bytes_target_field: 1024 } | ||
# required pipeline via scripted upsert | ||
- do: | ||
update: | ||
index: test | ||
id: 4 | ||
body: | ||
script: | ||
source: "ctx._source.bytes_source_field = '1kb'" | ||
lang: "painless" | ||
upsert : {} | ||
scripted_upsert: true | ||
- do: | ||
get: | ||
index: test | ||
id: 4 | ||
- match: { _source.bytes_source_field: "1kb" } | ||
- match: { _source.bytes_target_field: 1024 } | ||
# required pipeline via doc_as_upsert | ||
- do: | ||
update: | ||
index: test | ||
id: 5 | ||
body: | ||
doc: { "bytes_source_field":"1kb" } | ||
doc_as_upsert: true | ||
- do: | ||
get: | ||
index: test | ||
id: 5 | ||
- match: { _source.bytes_source_field: "1kb" } | ||
- match: { _source.bytes_target_field: 1024 } | ||
# required pipeline via bulk upsert | ||
# note - bulk scripted upsert's execute the pipeline before the script, so any data referenced by the pipeline | ||
# needs to be in the upsert, not the script | ||
- do: | ||
bulk: | ||
refresh: true | ||
body: | | ||
{"update":{"_id":"6","_index":"test"}} | ||
{"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}} | ||
{"update":{"_id":"7","_index":"test"}} | ||
{"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true} | ||
{"update":{"_id":"8","_index":"test"}} | ||
{"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true} | ||
{"update":{"_id":"6_alias","_index":"test_alias"}} | ||
{"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}} | ||
{"update":{"_id":"7_alias","_index":"test_alias"}} | ||
{"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true} | ||
{"update":{"_id":"8_alias","_index":"test_alias"}} | ||
{"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true} | ||
- do: | ||
mget: | ||
body: | ||
docs: | ||
- { _index: "test", _id: "6" } | ||
- { _index: "test", _id: "7" } | ||
- { _index: "test", _id: "8" } | ||
- { _index: "test", _id: "6_alias" } | ||
- { _index: "test", _id: "7_alias" } | ||
- { _index: "test", _id: "8_alias" } | ||
- match: { docs.0._index: "test" } | ||
- match: { docs.0._id: "6" } | ||
- match: { docs.0._source.bytes_source_field: "1kb" } | ||
- match: { docs.0._source.bytes_target_field: 1024 } | ||
- is_false: docs.0._source.ran_script | ||
- match: { docs.1._index: "test" } | ||
- match: { docs.1._id: "7" } | ||
- match: { docs.1._source.bytes_source_field: "2kb" } | ||
- match: { docs.1._source.bytes_target_field: 2048 } | ||
- match: { docs.2._index: "test" } | ||
- match: { docs.2._id: "8" } | ||
- match: { docs.2._source.bytes_source_field: "3kb" } | ||
- match: { docs.2._source.bytes_target_field: 3072 } | ||
- match: { docs.2._source.ran_script: true } | ||
- match: { docs.3._index: "test" } | ||
- match: { docs.3._id: "6_alias" } | ||
- match: { docs.3._source.bytes_source_field: "1kb" } | ||
- match: { docs.3._source.bytes_target_field: 1024 } | ||
- is_false: docs.3._source.ran_script | ||
- match: { docs.4._index: "test" } | ||
- match: { docs.4._id: "7_alias" } | ||
- match: { docs.4._source.bytes_source_field: "2kb" } | ||
- match: { docs.4._source.bytes_target_field: 2048 } | ||
- match: { docs.5._index: "test" } | ||
- match: { docs.5._id: "8_alias" } | ||
- match: { docs.5._source.bytes_source_field: "3kb" } | ||
- match: { docs.5._source.bytes_target_field: 3072 } | ||
- match: { docs.5._source.ran_script: true } | ||
|
||
# bad request, request pipeline can not be specified | ||
- do: | ||
catch: /illegal_argument_exception.*request pipeline \[pipeline\] can not override required pipeline \[my_pipeline\]/ | ||
index: | ||
index: test | ||
id: 9 | ||
pipeline: "pipeline" | ||
body: {bytes_source_field: "1kb"} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
133 changes: 133 additions & 0 deletions
133
server/src/test/java/org/elasticsearch/index/RequiredPipelineIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,133 @@ | ||
/* | ||
* Licensed to Elasticsearch under one or more contributor | ||
* license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright | ||
* ownership. Elasticsearch licenses this file to you under | ||
* the Apache License, Version 2.0 (the "License"); you may | ||
* not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
package org.elasticsearch.index; | ||
|
||
import org.elasticsearch.action.index.IndexRequestBuilder; | ||
import org.elasticsearch.common.settings.Settings; | ||
import org.elasticsearch.test.ESIntegTestCase; | ||
|
||
import java.io.IOException; | ||
import java.util.Collections; | ||
import java.util.Map; | ||
|
||
import static org.hamcrest.Matchers.containsString; | ||
import static org.hamcrest.Matchers.hasToString; | ||
|
||
public class RequiredPipelineIT extends ESIntegTestCase { | ||
|
||
public void testRequiredPipeline() { | ||
final Settings settings = Settings.builder().put(IndexSettings.REQUIRED_PIPELINE.getKey(), "required_pipeline").build(); | ||
createIndex("index", settings); | ||
|
||
// this asserts that the required_pipeline was used, without us having to actually create the pipeline etc. | ||
final IllegalArgumentException e = expectThrows( | ||
IllegalArgumentException.class, | ||
() -> client().prepareIndex("index", "_doc", "1").setSource(Collections.singletonMap("field", "value")).get()); | ||
assertThat(e, hasToString(containsString("pipeline with id [required_pipeline] does not exist"))); | ||
} | ||
|
||
public void testDefaultAndRequiredPipeline() { | ||
final Settings settings = Settings.builder() | ||
.put(IndexSettings.DEFAULT_PIPELINE.getKey(), "default_pipeline") | ||
.put(IndexSettings.REQUIRED_PIPELINE.getKey(), "required_pipeline") | ||
.build(); | ||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> createIndex("index", settings)); | ||
assertThat( | ||
e, | ||
hasToString(containsString("index has a default pipeline [default_pipeline] and a required pipeline [required_pipeline]"))); | ||
} | ||
|
||
public void testDefaultAndRequiredPipelineFromTemplates() { | ||
final int lowOrder = randomIntBetween(0, Integer.MAX_VALUE - 1); | ||
final int highOrder = randomIntBetween(lowOrder + 1, Integer.MAX_VALUE); | ||
final int requiredPipelineOrder; | ||
final int defaultPipelineOrder; | ||
if (randomBoolean()) { | ||
defaultPipelineOrder = lowOrder; | ||
requiredPipelineOrder = highOrder; | ||
} else { | ||
defaultPipelineOrder = highOrder; | ||
requiredPipelineOrder = lowOrder; | ||
} | ||
final Settings defaultPipelineSettings = | ||
Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "default_pipeline").build(); | ||
admin().indices() | ||
.preparePutTemplate("default") | ||
.setPatterns(Collections.singletonList("index*")) | ||
.setOrder(defaultPipelineOrder) | ||
.setSettings(defaultPipelineSettings) | ||
.get(); | ||
final Settings requiredPipelineSettings = | ||
Settings.builder().put(IndexSettings.REQUIRED_PIPELINE.getKey(), "required_pipeline").build(); | ||
admin().indices() | ||
.preparePutTemplate("required") | ||
.setPatterns(Collections.singletonList("index*")) | ||
.setOrder(requiredPipelineOrder) | ||
.setSettings(requiredPipelineSettings) | ||
.get(); | ||
final IllegalArgumentException e = expectThrows( | ||
IllegalArgumentException.class, | ||
() -> client().prepareIndex("index", "_doc", "1").setSource(Collections.singletonMap("field", "value")).get()); | ||
assertThat( | ||
e, | ||
hasToString(containsString( | ||
"required pipeline [required_pipeline] and default pipeline [default_pipeline] can not both be set"))); | ||
} | ||
|
||
public void testHighOrderRequiredPipelinePreferred() throws IOException { | ||
final int lowOrder = randomIntBetween(0, Integer.MAX_VALUE - 1); | ||
final int highOrder = randomIntBetween(lowOrder + 1, Integer.MAX_VALUE); | ||
final Settings defaultPipelineSettings = | ||
Settings.builder().put(IndexSettings.REQUIRED_PIPELINE.getKey(), "low_order_required_pipeline").build(); | ||
admin().indices() | ||
.preparePutTemplate("default") | ||
.setPatterns(Collections.singletonList("index*")) | ||
.setOrder(lowOrder) | ||
.setSettings(defaultPipelineSettings) | ||
.get(); | ||
final Settings requiredPipelineSettings = | ||
Settings.builder().put(IndexSettings.REQUIRED_PIPELINE.getKey(), "high_order_required_pipeline").build(); | ||
admin().indices() | ||
.preparePutTemplate("required") | ||
.setPatterns(Collections.singletonList("index*")) | ||
.setOrder(highOrder) | ||
.setSettings(requiredPipelineSettings) | ||
.get(); | ||
|
||
// this asserts that the high_order_required_pipeline was selected, without us having to actually create the pipeline etc. | ||
final IllegalArgumentException e = expectThrows( | ||
IllegalArgumentException.class, | ||
() -> client().prepareIndex("index", "_doc", "1").setSource(Collections.singletonMap("field", "value")).get()); | ||
assertThat(e, hasToString(containsString("pipeline with id [high_order_required_pipeline] does not exist"))); | ||
} | ||
|
||
public void testRequiredPipelineAndRequestPipeline() { | ||
final Settings settings = Settings.builder().put(IndexSettings.REQUIRED_PIPELINE.getKey(), "required_pipeline").build(); | ||
createIndex("index", settings); | ||
final IndexRequestBuilder builder = client().prepareIndex("index", "_doc", "1"); | ||
builder.setSource(Collections.singletonMap("field", "value")); | ||
builder.setPipeline("request_pipeline"); | ||
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::get); | ||
assertThat( | ||
e, | ||
hasToString(containsString("request pipeline [request_pipeline] can not override required pipeline [required_pipeline]"))); | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters