diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index bb3c75f10aaea..beb45107bc313 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -62,7 +62,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 12729a9b6ebda..cd0bc8449f89e 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -594,8 +594,8 @@ steps: env: BWC_VERSION: 8.14.3 - - label: "{{matrix.image}} / 8.15.1 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.1 + - label: "{{matrix.image}} / 8.15.2 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.2 timeout_in_minutes: 300 matrix: setup: @@ -609,7 +609,7 @@ steps: buildDirectory: /dev/shm/bk diskSizeGb: 250 env: - BWC_VERSION: 8.15.1 + BWC_VERSION: 8.15.2 - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 740fec13d1790..8f25a0fb11065 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -662,8 +662,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.15.1 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.1#bwcTest + - label: 8.15.2 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.2#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -673,7 +673,7 @@ steps: preemptible: true diskSizeGb: 250 env: - BWC_VERSION: 8.15.1 + BWC_VERSION: 8.15.2 retry: automatic: - exit_status: "-1" @@ -771,7 +771,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -821,7 +821,7 @@ steps: - openjdk21 - openjdk22 - openjdk23 - BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index e43b3333dd755..b80309cdb3f8e 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -32,5 +32,5 @@ BWC_VERSION: - "8.12.2" - "8.13.4" - "8.14.3" - - "8.15.1" + - "8.15.2" - "8.16.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 2eea118e57e2a..e41bbac68f1ec 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,4 @@ BWC_VERSION: - "7.17.24" - - "8.15.1" + - "8.15.2" - "8.16.0" diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy index b7c4908e39b62..737c448f23be6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy @@ -55,8 +55,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build() then: - // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet - result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED + result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE result.task(transformTask).outcome == TaskOutcome.NO_SOURCE @@ -165,7 +164,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe then: result.task(':check').outcome == TaskOutcome.UP_TO_DATE result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE - result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED + result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE result.task(transformTask).outcome == TaskOutcome.NO_SOURCE diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index a6dae60ddd524..d604973efcb4b 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -41,7 +41,7 @@ develocity { if (BuildParams.inFipsJvm) { tag 'FIPS' } - println "onCI = $onCI" + if (onCI) { //Buildkite-specific build scan metadata String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH') diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java index c6320394ef5b9..e0581ebf67081 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java @@ -35,6 +35,7 @@ import org.gradle.api.tasks.Sync; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; +import org.gradle.language.jvm.tasks.ProcessResources; import java.io.File; import java.nio.file.Path; @@ -213,6 +214,17 @@ public void apply(Project project) { .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK) .flatMap(CopyRestTestsTask::getOutputResourceDir); + // ensure we include other non rest spec related test resources + project.getTasks() + .withType(ProcessResources.class) + .named(yamlCompatTestSourceSet.getProcessResourcesTaskName()) + .configure(processResources -> { + processResources.from( + sourceSets.getByName(YamlRestTestPlugin.YAML_REST_TEST).getResources(), + spec -> { spec.exclude("rest-api-spec/**"); } + ); + }); + // setup the test task TaskProvider yamlRestCompatTestTask = registerTestTask(project, yamlCompatTestSourceSet); yamlRestCompatTestTask.configure(testTask -> { @@ -221,7 +233,7 @@ public void apply(Project project) { testTask.setTestClassesDirs( yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs()) ); - testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false); + testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getOutput().isEmpty() == false); testTask.setClasspath( yamlCompatTestSourceSet.getRuntimeClasspath() // remove the "normal" api and tests diff --git a/docs/changelog/111161.yaml b/docs/changelog/111161.yaml new file mode 100644 index 0000000000000..c081d555ff1ee --- /dev/null +++ b/docs/changelog/111161.yaml @@ -0,0 +1,6 @@ +pr: 111161 +summary: Add support for templates when validating mappings in the simulate ingest + API +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/changelog/111285.yaml b/docs/changelog/111285.yaml deleted file mode 100644 index e4856482b4d6e..0000000000000 --- a/docs/changelog/111285.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111285 -summary: "[Bugfix] Add `accessDeclaredMembers` permission to allow search application templates to parse floats" -area: Relevance -type: bug -issues: [] diff --git a/docs/changelog/111475.yaml b/docs/changelog/111475.yaml deleted file mode 100644 index 264c975444868..0000000000000 --- a/docs/changelog/111475.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111475 -summary: "ESQL: Fix for overzealous validation in case of invalid mapped fields" -area: ES|QL -type: bug -issues: - - 111452 diff --git a/docs/changelog/111548.yaml b/docs/changelog/111548.yaml new file mode 100644 index 0000000000000..ca9e5ae622894 --- /dev/null +++ b/docs/changelog/111548.yaml @@ -0,0 +1,6 @@ +pr: 111548 +summary: Json parsing exceptions should not cause 500 errors +area: Infra/Core +type: bug +issues: + - 111542 diff --git a/docs/changelog/111673.yaml b/docs/changelog/111673.yaml deleted file mode 100644 index ebc211633fcab..0000000000000 --- a/docs/changelog/111673.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111673 -summary: Properly handle filters on `TextSimilarityRank` retriever -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/111729.yaml b/docs/changelog/111729.yaml deleted file mode 100644 index c75c14a997da9..0000000000000 --- a/docs/changelog/111729.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111729 -summary: Speed up dense/sparse vector stats -area: Vector Search -type: bug -issues: - - 111715 diff --git a/docs/changelog/111756.yaml b/docs/changelog/111756.yaml deleted file mode 100644 index e58345dbe696a..0000000000000 --- a/docs/changelog/111756.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111756 -summary: Fix `NullPointerException` when doing knn search on empty index without dims -area: Vector Search -type: bug -issues: - - 111733 diff --git a/docs/changelog/111758.yaml b/docs/changelog/111758.yaml deleted file mode 100644 index c95cdf48bc8a7..0000000000000 --- a/docs/changelog/111758.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111758 -summary: Revert "Avoid bucket copies in Aggs" -area: Aggregations -type: bug -issues: - - 111679 diff --git a/docs/changelog/111807.yaml b/docs/changelog/111807.yaml deleted file mode 100644 index 97c5e58461c34..0000000000000 --- a/docs/changelog/111807.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111807 -summary: Explain Function Score Query -area: Search -type: bug -issues: [] diff --git a/docs/changelog/111843.yaml b/docs/changelog/111843.yaml deleted file mode 100644 index c8b20036520f3..0000000000000 --- a/docs/changelog/111843.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111843 -summary: Add maximum nested depth check to WKT parser -area: Geo -type: bug -issues: [] diff --git a/docs/changelog/111863.yaml b/docs/changelog/111863.yaml deleted file mode 100644 index 1724cd83f984b..0000000000000 --- a/docs/changelog/111863.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111863 -summary: Fixing incorrect bulk request took time -area: Ingest Node -type: bug -issues: - - 111854 diff --git a/docs/changelog/111866.yaml b/docs/changelog/111866.yaml deleted file mode 100644 index 34bf56da4dc9e..0000000000000 --- a/docs/changelog/111866.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111866 -summary: Fix windows memory locking -area: Infra/Core -type: bug -issues: - - 111847 diff --git a/docs/changelog/111943.yaml b/docs/changelog/111943.yaml deleted file mode 100644 index 6b9f03ccee31c..0000000000000 --- a/docs/changelog/111943.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111943 -summary: Fix synthetic source for empty nested objects -area: Mapping -type: bug -issues: - - 111811 diff --git a/docs/changelog/111947.yaml b/docs/changelog/111947.yaml deleted file mode 100644 index 0aff0b9c7b8be..0000000000000 --- a/docs/changelog/111947.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111947 -summary: Improve performance of grok pattern cycle detection -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/111966.yaml b/docs/changelog/111966.yaml deleted file mode 100644 index facf0a61c4d8a..0000000000000 --- a/docs/changelog/111966.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111966 -summary: No error when `store_array_source` is used without synthetic source -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/111983.yaml b/docs/changelog/111983.yaml deleted file mode 100644 index d5043d0b44155..0000000000000 --- a/docs/changelog/111983.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111983 -summary: Avoid losing error message in failure collector -area: ES|QL -type: bug -issues: - - 111894 diff --git a/docs/changelog/111994.yaml b/docs/changelog/111994.yaml deleted file mode 100644 index ee62651c43987..0000000000000 --- a/docs/changelog/111994.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111994 -summary: Merge multiple ignored source entires for the same field -area: Logs -type: bug -issues: - - 111694 diff --git a/docs/changelog/112005.yaml b/docs/changelog/112005.yaml deleted file mode 100644 index 2d84381e632b3..0000000000000 --- a/docs/changelog/112005.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112005 -summary: Check for valid `parentDoc` before retrieving its previous -area: Mapping -type: bug -issues: - - 111990 diff --git a/docs/changelog/112038.yaml b/docs/changelog/112038.yaml deleted file mode 100644 index 6cbfb373b7420..0000000000000 --- a/docs/changelog/112038.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112038 -summary: Semantic reranking should fail whenever inference ID does not exist -area: Relevance -type: bug -issues: - - 111934 diff --git a/docs/changelog/112046.yaml b/docs/changelog/112046.yaml deleted file mode 100644 index f3cda1ed7a7d2..0000000000000 --- a/docs/changelog/112046.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112046 -summary: Fix calculation of parent offset for ignored source in some cases -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/112090.yaml b/docs/changelog/112090.yaml deleted file mode 100644 index 6d6e4d0851523..0000000000000 --- a/docs/changelog/112090.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112090 -summary: Always check `crsType` when folding spatial functions -area: Geo -type: bug -issues: - - 112089 diff --git a/docs/changelog/112135.yaml b/docs/changelog/112135.yaml deleted file mode 100644 index d2ff6994b6196..0000000000000 --- a/docs/changelog/112135.yaml +++ /dev/null @@ -1,4 +0,0 @@ -pr: 112135 -summary: Fix the bug where the run() function of ExecutableInferenceRequest throws an exception when get inferenceEntityId. -area: Inference -type: bug diff --git a/docs/changelog/112139.yaml b/docs/changelog/112139.yaml deleted file mode 100644 index d6d992ec1dcf2..0000000000000 --- a/docs/changelog/112139.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112139 -summary: Fix NPE when executing doc value queries over shape geometries with empty - segments -area: Geo -type: bug -issues: [] diff --git a/docs/changelog/112173.yaml b/docs/changelog/112173.yaml deleted file mode 100644 index 9a43b0d1bf1fa..0000000000000 --- a/docs/changelog/112173.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 112173 -summary: Prevent synthetic field loaders accessing stored fields from using stale - data -area: Mapping -type: bug -issues: - - 112156 diff --git a/docs/changelog/112178.yaml b/docs/changelog/112178.yaml deleted file mode 100644 index f1011291542b8..0000000000000 --- a/docs/changelog/112178.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112178 -summary: Avoid wrapping rejection exception in exchange -area: ES|QL -type: bug -issues: - - 112106 diff --git a/docs/changelog/112217.yaml b/docs/changelog/112217.yaml deleted file mode 100644 index bb367d6128001..0000000000000 --- a/docs/changelog/112217.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112217 -summary: Fix template alias parsing livelock -area: Indices APIs -type: bug -issues: [] diff --git a/docs/changelog/112226.yaml b/docs/changelog/112226.yaml deleted file mode 100644 index ac36c0c0fe4e2..0000000000000 --- a/docs/changelog/112226.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112226 -summary: "Fix \"unexpected field [remote_cluster]\" for CCS (RCS 1.0) when using API\ - \ key that references `remote_cluster`" -area: Security -type: bug -issues: [] diff --git a/docs/changelog/112230.yaml b/docs/changelog/112230.yaml deleted file mode 100644 index ef12dc3f78267..0000000000000 --- a/docs/changelog/112230.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112230 -summary: Fix connection timeout for `OpenIdConnectAuthenticator` get Userinfo -area: Security -type: bug -issues: [] diff --git a/docs/changelog/112242.yaml b/docs/changelog/112242.yaml deleted file mode 100644 index 7292a00166de2..0000000000000 --- a/docs/changelog/112242.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112242 -summary: Fix toReleaseVersion() when called on the current version id -area: Infra/Core -type: bug -issues: [111900] diff --git a/docs/changelog/112260.yaml b/docs/changelog/112260.yaml deleted file mode 100644 index 3f5642188a367..0000000000000 --- a/docs/changelog/112260.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112260 -summary: Fix DLS over Runtime Fields -area: "Authorization" -type: bug -issues: - - 111637 diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index 36f1f089ce90e..ee84a39ee6f65 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -119,7 +119,11 @@ as well the same way that a non-simulated ingest would. No data is indexed into {es}. Instead, the transformed document is returned, along with the list of pipelines that have been executed and the name of the index where the document would have been indexed if this were -not a simulation. This differs from the +not a simulation. The transformed document is validated against the +mappings that would apply to this index, and any validation error is +reported in the result. + +This API differs from the <> in that you specify a single pipeline for that API, and it only runs that one pipeline. The simulate pipeline API is more useful for developing a single pipeline, diff --git a/docs/reference/mapping/types/sparse-vector.asciidoc b/docs/reference/mapping/types/sparse-vector.asciidoc index d0c2c83b8a8fa..b24f65fcf97ca 100644 --- a/docs/reference/mapping/types/sparse-vector.asciidoc +++ b/docs/reference/mapping/types/sparse-vector.asciidoc @@ -91,7 +91,7 @@ NOTE: `sparse_vector` fields can not be included in indices that were *created* NOTE: `sparse_vector` fields only support strictly positive values. Negative values will be rejected. -NOTE: `sparse_vector` fields do not support querying, sorting or aggregating. +NOTE: `sparse_vector` fields do not support <>, querying, sorting or aggregating. They may only be used within specialized queries. The recommended query to use on these fields are <> queries. They may also be used within legacy <> queries. diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java index c8e429d4c1490..c59f003d9cb04 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java @@ -57,7 +57,8 @@ public Token nextToken() throws IOException { try { return convertToken(parser.nextToken()); } catch (JsonEOFException e) { - throw new XContentEOFException(e); + JsonLocation location = e.getLocation(); + throw new XContentEOFException(new XContentLocation(location.getLineNr(), location.getColumnNr()), "Unexpected end of file", e); } catch (JsonParseException e) { throw newXContentParseException(e); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java index de9ea6fb04f26..01a2407598159 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java @@ -8,11 +8,9 @@ package org.elasticsearch.xcontent; -import java.io.IOException; +public class XContentEOFException extends XContentParseException { -public class XContentEOFException extends IOException { - - public XContentEOFException(IOException cause) { - super(cause); + public XContentEOFException(XContentLocation location, String message, Exception cause) { + super(location, message, cause); } } diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index a773c751eeaf5..91f3303d9d4a8 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -54,6 +54,9 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Filtered test", "Hybrid t-digest produces different results.") task.skipTest("search.aggregation/420_percentile_ranks_tdigest_metric/filtered", "Hybrid t-digest produces different results.") + // Something has changed with response codes + task.skipTest("search.aggregation/20_terms/IP test", "Hybrid t-digest produces different results.") + task.addAllowedWarningRegex("\\[types removal\\].*") } diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index a0375c61d7c29..daf0c188cc83e 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.internal-cluster-test' @@ -23,11 +24,7 @@ dependencies { internalClusterTestImplementation project(":modules:mapper-extras") } -tasks.named('yamlRestTest') { - usesDefaultDistribution() -} - -tasks.named('javaRestTest') { +tasks.withType(StandaloneRestIntegTestTask).configureEach { usesDefaultDistribution() } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index c3fd479616319..29cda588bc26b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestUtils; @@ -61,17 +62,19 @@ public Set supportedCapabilities() { @Override public Set supportedQueryParameters() { - return Set.of( - "name", - "include_defaults", - "timeout", - "master_timeout", - RestRequest.PATH_RESTRICTED, - IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, - IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, - IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, - IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, - "verbose" + return Sets.union( + RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, + Set.of( + "name", + "include_defaults", + "timeout", + "master_timeout", + IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, + IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, + IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, + IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, + "verbose" + ) ); } } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 90d52de6f0fff..d7100745680ba 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -5,6 +5,8 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' @@ -29,7 +31,7 @@ restResources { } } -tasks.named('yamlRestTest') { +tasks.withType(StandaloneRestIntegTestTask).configureEach { usesDefaultDistribution() } diff --git a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml index 109bc8888889f..de9b3a0ec9bc2 100644 --- a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml +++ b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml @@ -114,14 +114,14 @@ setup: - match: { responses.0.hits.total: 2 } - match: { responses.1.error.root_cause.0.type: x_content_e_o_f_exception } - - match: { responses.1.error.root_cause.0.reason: "/Unexpected.end.of.input/" } + - match: { responses.1.error.root_cause.0.reason: "/\\[1:22\\].Unexpected.end.of.file/" } - match: { responses.2.hits.total: 1 } - match: { responses.3.error.root_cause.0.type: parsing_exception } - match: { responses.3.error.root_cause.0.reason: "/unknown.query.\\[unknown\\]/" } - match: { responses.4.error.root_cause.0.type: illegal_argument_exception } - match: { responses.4.error.root_cause.0.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" } - match: { responses.0.status: 200 } - - match: { responses.1.status: 500 } + - match: { responses.1.status: 400 } - match: { responses.2.status: 200 } - match: { responses.3.status: 400 } diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 3537d430e212b..3fe2f9d9bae42 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -33,6 +33,11 @@ dependencies { internalClusterTestImplementation project(':test:fixtures:url-fixture') } +tasks.named("yamlRestTestV7CompatTransform").configure { task -> + task.skipTest("repository_url/10_basic/Restore with repository-url using file://", "Error message has changed") + task.skipTest("repository_url/10_basic/Restore with repository-url using http://", "Error message has changed") +} + tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( 'javax.servlet.ServletContextEvent', diff --git a/muted-tests.yml b/muted-tests.yml index 58d3060c90ad8..a8136219b3da2 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -170,6 +170,14 @@ tests: - class: org.elasticsearch.xpack.security.authc.kerberos.SimpleKdcLdapServerTests method: testClientServiceMutualAuthentication issue: https://github.com/elastic/elasticsearch/issues/112529 +- class: org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/111497 +- class: org.elasticsearch.smoketest.SmokeTestIngestWithAllDepsClientYamlTestSuiteIT + method: test {yaml=ingest/80_ingest_simulate/Test ingest simulate with reroute and mapping validation from templates} + issue: https://github.com/elastic/elasticsearch/issues/112575 +- class: org.elasticsearch.script.mustache.LangMustacheClientYamlTestSuiteIT + method: test {yaml=lang_mustache/50_multi_search_template/Multi-search template with errors} + issue: https://github.com/elastic/elasticsearch/issues/112580 # Examples: # diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index a42b987a9bddd..1a77019914283 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -212,3 +212,88 @@ setup: - match: { docs.1.doc._index: "index" } - match: { docs.1.doc._source.field1: "BAR" } - match: { docs.1.doc.executed_pipelines: ["my-pipeline"] } + +--- +"Test ingest simulate with reroute and mapping validation from templates": + + - skip: + features: headers + + - requires: + cluster_features: ["simulate.mapping.validation.templates"] + reason: "ingest simulate index mapping validation added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "reroute-pipeline" + body: > + { + "processors": [ + { + "reroute": { + "destination": "second-index" + } + } + ] + } + - match: { acknowledged: true } + + - do: + indices.put_index_template: + name: first-index-template + body: + index_patterns: first-index* + template: + settings: + default_pipeline: "reroute-pipeline" + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + indices.put_index_template: + name: second-index-template + body: + index_patterns: second-index* + template: + mappings: + dynamic: strict + properties: + bar: + type: text + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "first-index", + "_id": "id", + "_source": { + "foo": "bar" + } + }, + { + "_index": "first-index", + "_id": "id", + "_source": { + "bar": "foo" + } + } + ] + } + - length: { docs: 2 } + - match: { docs.0.doc._index: "second-index" } + - match: { docs.0.doc._source.foo: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:8] mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed" } + - match: { docs.1.doc._index: "second-index" } + - match: { docs.1.doc._source.bar: "foo" } + - not_exists: docs.1.doc.error diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json index 4ea78bfd45460..e27e3a0450bff 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json @@ -37,6 +37,16 @@ ], "default":"open", "description":"Whether wildcard expressions should get expanded to open or closed indices (default: open)" + }, + "ignore_unavailable":{ + "type":"boolean", + "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)", + "default":false + }, + "allow_no_indices":{ + "type":"boolean", + "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)", + "default":true } } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml index 5928dce2c104e..a32969b0b69b2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml @@ -258,6 +258,274 @@ setup: - not_exists: docs.1.doc.error --- +"Test mapping validation from templates": + + - skip: + features: headers + + - requires: + cluster_features: ["simulate.mapping.validation.templates"] + reason: "ingest simulate index mapping validation added in 8.16" + + - do: + indices.put_template: + name: v1_template + body: + index_patterns: v1_strict_nonexistent* + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + indices.put_index_template: + name: v2_template + body: + index_patterns: v2_strict_nonexistent* + template: + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + indices.put_index_template: + name: v2_hidden_template + body: + index_patterns: v2_strict_hidden_nonexistent* + template: + settings: + index: + hidden: true + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "v1_strict_nonexistent_index", + "_id": "id", + "_source": { + "foob": "bar" + } + }, + { + "_index": "v1_strict_nonexistent_index", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 2 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:9] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "v2_strict_nonexistent_index", + "_id": "id", + "_source": { + "foob": "bar" + } + }, + { + "_index": "v2_strict_nonexistent_index", + "_id": "id", + "_source": { + "foo": "rab" + } + }, + { + "_index": "v2_strict_hidden_nonexistent_index", + "_id": "id", + "_source": { + "foob": "bar" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 3 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:9] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error + - match: { docs.2.doc._source.foob: "bar" } + - match: { docs.2.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.2.doc.error.reason: "[1:9] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + +--- +"Test mapping validation for data streams from templates": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.mapping.validation.templates"] + reason: "ingest simulate index mapping validation added in 8.16" + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + settings: + index.number_of_replicas: 1 + mappings: + dynamic: strict + properties: + foo: + type: text + data_stream: {} + + - do: + allowed_warnings: + - "index template [my-hidden-template1] has index patterns [simple-hidden-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-hidden-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-hidden-template1 + body: + index_patterns: [simple-hidden-data-stream1] + template: + settings: + index.number_of_replicas: 1 + mappings: + dynamic: strict + properties: + foo: + type: text + data_stream: + hidden: true + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foob": "bar" + } + }, + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foo": "rab" + } + }, + { + "_index": "simple-hidden-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foob": "bar" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 3 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:35] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error + - match: { docs.2.doc._source.foob: "bar" } + - match: { docs.2.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.2.doc.error.reason: "[1:35] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foob": "bar" + } + }, + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foo": "rab" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 2 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:35] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error +--- "Test index templates with pipelines": - skip: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java new file mode 100644 index 0000000000000..4a56a6ce8ddb6 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -0,0 +1,278 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.ingest.SimulateIndexResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class TransportSimulateBulkActionIT extends ESIntegTestCase { + @SuppressWarnings("unchecked") + public void testMappingValidationIndexExists() { + /* + * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and + * that the index's mapping in the cluster state has not been updated with the two new field. + */ + String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + String mapping = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); + SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + searchResponse.decRef(); + ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest()).actionGet(); + Map indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap(); + Map fields = (Map) indexMapping.get("properties"); + assertThat(fields.size(), equalTo(1)); + } + + public void testMappingValidationIndexDoesNotExistsNoTemplate() { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. There is no template (other than the + * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. + */ + String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + + public void testMappingValidationIndexDoesNotExistsV2Template() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. The index matches a v2 index template. It + * has strict mappings and one of our documents has it as a field not in the mapping, so we expect a mapping validation error. + */ + String indexName = "my-index-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + String mappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(mappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .template(template) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + + public void testMappingValidationIndexDoesNotExistsV1Template() { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. The index matches a v1 index template. It + * has a mapping that defines "foo1" as an integer field and one of our documents has it as a string, so we expect a mapping + * validation exception. + */ + String indexName = "my-index-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + indicesAdmin().putTemplate( + new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") + ).actionGet(); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[0].getResponse()).getException().getMessage(), + containsString("failed to parse field [foo1] of type [integer] ") + ); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + } + + public void testMappingValidationIndexDoesNotExistsDataStream() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. The index matches a v2 index template. It + * has strict mappings and one of our documents has it as a field not in the mapping, so we expect a mapping validation error. + */ + String indexName = "my-data-stream-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + String mappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(mappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate(); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-data-stream-*")) + .dataStreamTemplate(dataStreamTemplate) + .template(template) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + { + // First, try with no @timestamp to make sure we're picking up data-stream-specific templates + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[0].getResponse()).getException().getMessage(), + containsString("data stream timestamp field [@timestamp] is missing") + ); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + { + // Now with @timestamp + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "@timestamp": "2024-08-27", + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "@timestamp": "2024-08-27", + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java index bb18b8f1b702d..8b7f69df9fcc3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java @@ -32,6 +32,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.query.SlowRunningQueryBuilder; import org.elasticsearch.search.query.ThrowingQueryBuilder; +import org.elasticsearch.search.retriever.MinimalCompoundRetrieverIT; +import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -49,6 +51,7 @@ import java.lang.annotation.Target; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -620,6 +623,20 @@ public void testPITSearch() throws ExecutionException, InterruptedException { assertThat(telemetry.getSuccessCount(), equalTo(2L)); } + public void testCompoundRetrieverSearch() throws ExecutionException, InterruptedException { + RetrieverBuilder compoundRetriever = new MinimalCompoundRetrieverIT.CompoundRetriever(Collections.emptyList()); + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + searchRequest.source(new SearchSourceBuilder().retriever(compoundRetriever)); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + } + private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) { var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName); return usage.getCcsUsageHolder().getCCSTelemetrySnapshot(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java new file mode 100644 index 0000000000000..8c65d28711c1b --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.retriever; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.equalTo; + +public class MinimalCompoundRetrieverIT extends AbstractMultiClustersTestCase { + + // CrossClusterSearchIT + private static final String REMOTE_CLUSTER = "cluster_a"; + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER); + } + + @Override + protected Map skipUnavailableForRemoteClusters() { + return Map.of(REMOTE_CLUSTER, randomBoolean()); + } + + @Override + protected boolean reuseClusters() { + return false; + } + + public void testSimpleSearch() throws ExecutionException, InterruptedException { + RetrieverBuilder compoundRetriever = new CompoundRetriever(Collections.emptyList()); + Map testClusterInfo = setupTwoClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + SearchRequest searchRequest = new SearchRequest(localIndex, REMOTE_CLUSTER + ":" + remoteIndex); + searchRequest.source(new SearchSourceBuilder().retriever(compoundRetriever)); + assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> { + assertNotNull(response); + + SearchResponse.Clusters clusters = response.getClusters(); + assertFalse("search cluster results should NOT be marked as partial", clusters.hasPartialResults()); + assertThat(clusters.getTotal(), equalTo(2)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.SUCCESSFUL), equalTo(2)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.SKIPPED), equalTo(0)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.RUNNING), equalTo(0)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.PARTIAL), equalTo(0)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.FAILED), equalTo(0)); + assertThat(response.getHits().getTotalHits().value, equalTo(testClusterInfo.get("total_docs"))); + }); + } + + private Map setupTwoClusters() { + int totalDocs = 0; + String localIndex = "demo"; + int numShardsLocal = randomIntBetween(2, 10); + Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); + assertAcked( + client(LOCAL_CLUSTER).admin() + .indices() + .prepareCreate(localIndex) + .setSettings(localSettings) + .setMapping("some_field", "type=keyword") + ); + totalDocs += indexDocs(client(LOCAL_CLUSTER), localIndex); + + String remoteIndex = "prod"; + int numShardsRemote = randomIntBetween(2, 10); + final InternalTestCluster remoteCluster = cluster(REMOTE_CLUSTER); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3)); + assertAcked( + client(REMOTE_CLUSTER).admin() + .indices() + .prepareCreate(remoteIndex) + .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) + .setMapping("some_field", "type=keyword") + ); + assertFalse( + client(REMOTE_CLUSTER).admin() + .cluster() + .prepareHealth(remoteIndex) + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(10)) + .get() + .isTimedOut() + ); + totalDocs += indexDocs(client(REMOTE_CLUSTER), remoteIndex); + + String skipUnavailableKey = Strings.format("cluster.remote.%s.skip_unavailable", REMOTE_CLUSTER); + Setting skipUnavailableSetting = cluster(REMOTE_CLUSTER).clusterService().getClusterSettings().get(skipUnavailableKey); + boolean skipUnavailable = (boolean) cluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY).clusterService() + .getClusterSettings() + .get(skipUnavailableSetting); + + Map clusterInfo = new HashMap<>(); + clusterInfo.put("local.num_shards", numShardsLocal); + clusterInfo.put("local.index", localIndex); + clusterInfo.put("remote.num_shards", numShardsRemote); + clusterInfo.put("remote.index", remoteIndex); + clusterInfo.put("remote.skip_unavailable", skipUnavailable); + clusterInfo.put("total_docs", (long) totalDocs); + return clusterInfo; + } + + private int indexDocs(Client client, String index) { + int numDocs = between(500, 1200); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("some_field", i).get(); + } + client.admin().indices().prepareRefresh(index).get(); + return numDocs; + } + + public static class CompoundRetriever extends RetrieverBuilder { + + private final List sources; + + public CompoundRetriever(List sources) { + this.sources = sources; + } + + @Override + public boolean isCompound() { + return true; + } + + @Override + public QueryBuilder topDocsQuery() { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { + if (ctx.getPointInTimeBuilder() == null) { + throw new IllegalStateException("PIT is required"); + } + if (sources.isEmpty()) { + StandardRetrieverBuilder standardRetrieverBuilder = new StandardRetrieverBuilder(); + standardRetrieverBuilder.queryBuilder = new MatchAllQueryBuilder(); + return standardRetrieverBuilder; + } + return sources.get(0); + } + + @Override + public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public String getName() { + return "compound_retriever"; + } + + @Override + protected void doToXContent(XContentBuilder builder, Params params) throws IOException { + // no-op + } + + @Override + protected boolean doEquals(Object o) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index b751daf0e2d98..54b6b1ef9c8c8 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -182,6 +182,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_14_3 = new Version(8_14_03_99); public static final Version V_8_15_0 = new Version(8_15_00_99); public static final Version V_8_15_1 = new Version(8_15_01_99); + public static final Version V_8_15_2 = new Version(8_15_02_99); public static final Version V_8_16_0 = new Version(8_16_00_99); public static final Version CURRENT = V_8_16_0; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 6fcaad47e0d72..cd8ffea3d3824 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -249,13 +250,20 @@ public static Template resolveTemplate( .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - // empty request mapping as the user can't specify any explicit mappings via the simulate api + /* + * If the index name doesn't look like a data stream backing index, then MetadataCreateIndexService.collectV2Mappings() won't + * include data stream specific mappings in its response. + */ + String simulatedIndexName = template.getDataStreamTemplate() != null + && indexName.startsWith(DataStream.BACKING_INDEX_PREFIX) == false + ? DataStream.getDefaultBackingIndexName(indexName, 1) + : indexName; List mappings = MetadataCreateIndexService.collectV2Mappings( - null, + null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, xContentRegistry, - indexName + simulatedIndexName ); // First apply settings sourced from index settings providers @@ -303,7 +311,9 @@ public static Template resolveTemplate( ) ); - Map aliasesByName = aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity())); + Map aliasesByName = aliases == null + ? Map.of() + : aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity())); CompressedXContent mergedMapping = indicesService.withTempIndexService( indexMetadata, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index 99c2d994a8bd0..b8dd0d1fe415e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -14,9 +14,10 @@ import java.util.Set; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION); + return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 2312a75b91084..8da6fb409cb90 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -10,16 +10,27 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -35,9 +46,18 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; +import static java.util.stream.Collectors.toList; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; + /** * This action simulates bulk indexing data. Pipelines are executed for all indices that the request routes to, but no data is actually * indexed and no state is changed. Unlike TransportBulkAction, this does not push the work out to the nodes where the shards live (since @@ -45,7 +65,10 @@ */ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation"); + public static final NodeFeature SIMULATE_MAPPING_VALIDATION_TEMPLATES = new NodeFeature("simulate.mapping.validation.templates"); private final IndicesService indicesService; + private final NamedXContentRegistry xContentRegistry; + private final Set indexSettingProviders; @Inject public TransportSimulateBulkAction( @@ -56,7 +79,9 @@ public TransportSimulateBulkAction( ActionFilters actionFilters, IndexingPressure indexingPressure, SystemIndices systemIndices, - IndicesService indicesService + IndicesService indicesService, + NamedXContentRegistry xContentRegistry, + IndexSettingProviders indexSettingProviders ) { super( SimulateBulkAction.INSTANCE, @@ -71,6 +96,8 @@ public TransportSimulateBulkAction( threadPool::relativeTimeInNanos ); this.indicesService = indicesService; + this.xContentRegistry = xContentRegistry; + this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); } @Override @@ -128,9 +155,9 @@ private Exception validateMappings(IndexRequest request) { ClusterState state = clusterService.state(); Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); - if (indexAbstraction != null) { - IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); - try { + try { + if (indexAbstraction != null) { + IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); return IndexShard.prepareIndex( @@ -148,9 +175,102 @@ private Exception validateMappings(IndexRequest request) { 0 ); }); - } catch (Exception e) { - mappingValidationException = e; + } else { + /* + * The index did not exist, so we put together the mappings from existing templates. + * This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). However, + * it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the path for + * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate + * writing to system indices. + */ + String matchingTemplate = findV2Template(state.metadata(), request.index(), false); + if (matchingTemplate != null) { + final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( + matchingTemplate, + request.index(), + state, + isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), + xContentRegistry, + indicesService, + systemIndices, + indexSettingProviders + ); + CompressedXContent mappings = template.mappings(); + if (mappings != null) { + MappingMetadata mappingMetadata = new MappingMetadata(mappings); + Settings dummySettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + final IndexMetadata imd = IndexMetadata.builder(request.index()) + .settings(dummySettings) + .putMapping(mappingMetadata) + .build(); + indicesService.withTempIndexService(imd, indexService -> { + indexService.mapperService().updateMapping(null, imd); + return IndexShard.prepareIndex( + indexService.mapperService(), + sourceToParse, + SequenceNumbers.UNASSIGNED_SEQ_NO, + -1, + -1, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + Long.MIN_VALUE, + false, + request.ifSeqNo(), + request.ifPrimaryTerm(), + 0 + ); + }); + } + } else { + List matchingTemplates = findV1Templates(state.metadata(), request.index(), false); + final Map mappingsMap = MetadataCreateIndexService.parseV1Mappings( + "{}", + matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), + xContentRegistry + ); + final CompressedXContent combinedMappings; + if (mappingsMap.isEmpty()) { + combinedMappings = null; + } else { + combinedMappings = new CompressedXContent(mappingsMap); + } + Settings dummySettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + MappingMetadata mappingMetadata = combinedMappings == null ? null : new MappingMetadata(combinedMappings); + final IndexMetadata imd = IndexMetadata.builder(request.index()) + .putMapping(mappingMetadata) + .settings(dummySettings) + .build(); + indicesService.withTempIndexService(imd, indexService -> { + indexService.mapperService().updateMapping(null, imd); + return IndexShard.prepareIndex( + indexService.mapperService(), + sourceToParse, + SequenceNumbers.UNASSIGNED_SEQ_NO, + -1, + -1, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + Long.MIN_VALUE, + false, + request.ifSeqNo(), + request.ifPrimaryTerm(), + 0 + ); + }); + } } + } catch (Exception e) { + mappingValidationException = e; } return mappingValidationException; } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java index 445492f037926..258cd5ceaa8e7 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java @@ -96,6 +96,10 @@ public void writeTo(StreamOutput out) throws IOException { } } + public Exception getException() { + return this.exception; + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index e29b07eeffe11..23ff692da4887 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -312,8 +312,12 @@ public long buildTookInMillis() { @Override protected void doExecute(Task task, SearchRequest searchRequest, ActionListener listener) { - ActionListener loggingAndMetrics = new SearchResponseActionListener((SearchTask) task, listener); - executeRequest((SearchTask) task, searchRequest, loggingAndMetrics, AsyncSearchActionProvider::new); + executeRequest( + (SearchTask) task, + searchRequest, + new SearchResponseActionListener((SearchTask) task, listener), + AsyncSearchActionProvider::new + ); } void executeRequest( @@ -498,7 +502,7 @@ void executeRequest( // We set the keep alive to -1 to indicate that we don't need the pit id in the response. // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore. source.pointInTimeBuilder(new PointInTimeBuilder(resp.getPointInTimeId()).setKeepAlive(TimeValue.MINUS_ONE)); - executeRequest(task, original, new ActionListener<>() { + var pitListener = new SearchResponseActionListener(task, listener) { @Override public void onResponse(SearchResponse response) { // we need to close the PIT first so we delay the release of the response to after the closing @@ -514,7 +518,8 @@ public void onResponse(SearchResponse response) { public void onFailure(Exception e) { closePIT(client, original.source().pointInTimeBuilder(), () -> listener.onFailure(e)); } - }, searchPhaseProvider); + }; + executeRequest(task, original, pitListener, searchPhaseProvider); })); } else { Rewriteable.rewriteAndFetch( @@ -1846,7 +1851,11 @@ private class SearchResponseActionListener implements ActionListener listener) { this.task = task; this.listener = listener; - usageBuilder = new CCSUsage.Builder(); + if (listener instanceof SearchResponseActionListener srListener) { + usageBuilder = srListener.usageBuilder; + } else { + usageBuilder = new CCSUsage.Builder(); + } } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 30e9a9a3779d7..02d5bdfdbebc0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -984,7 +984,9 @@ public ClusterState build() { routingTable, nodes, compatibilityVersions, - new ClusterFeatures(nodeFeatures), + previous != null && getNodeFeatures(previous.clusterFeatures).equals(nodeFeatures) + ? previous.clusterFeatures + : new ClusterFeatures(nodeFeatures), blocks, customs.build(), fromDiff, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 02b7312b4a99d..17db4f9253824 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -902,7 +902,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( * {@link IndexTemplateMetadata#order()}). This merging makes no distinction between field * definitions, as may result in an invalid field definition */ - static Map parseV1Mappings( + public static Map parseV1Mappings( String mappingsJson, List templateMappings, NamedXContentRegistry xContentRegistry diff --git a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java index cc908cd7cad2c..42a2854350fdb 100644 --- a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java @@ -157,30 +157,29 @@ public PersistentTasksCustomMetadata.Assignment getAssignment( // visible for testing void startTask(ClusterChangedEvent event) { // Wait until every node in the cluster supports health checks - if (event.state().clusterRecovered() && featureService.clusterHasFeature(event.state(), HealthFeatures.SUPPORTS_HEALTH)) { - boolean healthNodeTaskExists = HealthNode.findTask(event.state()) != null; - boolean isElectedMaster = event.localNodeMaster(); - if (isElectedMaster && healthNodeTaskExists == false) { - persistentTasksService.sendStartRequest( - TASK_NAME, - TASK_NAME, - new HealthNodeTaskParams(), - null, - ActionListener.wrap(r -> logger.debug("Created the health node task"), e -> { - if (e instanceof NodeClosedException) { - logger.debug("Failed to create health node task because node is shutting down", e); - return; + if (event.localNodeMaster() + && event.state().clusterRecovered() + && HealthNode.findTask(event.state()) == null + && featureService.clusterHasFeature(event.state(), HealthFeatures.SUPPORTS_HEALTH)) { + persistentTasksService.sendStartRequest( + TASK_NAME, + TASK_NAME, + new HealthNodeTaskParams(), + null, + ActionListener.wrap(r -> logger.debug("Created the health node task"), e -> { + if (e instanceof NodeClosedException) { + logger.debug("Failed to create health node task because node is shutting down", e); + return; + } + Throwable t = e instanceof RemoteTransportException ? e.getCause() : e; + if (t instanceof ResourceAlreadyExistsException == false) { + logger.error("Failed to create the health node task", e); + if (enabled) { + clusterService.addListener(taskStarter); } - Throwable t = e instanceof RemoteTransportException ? e.getCause() : e; - if (t instanceof ResourceAlreadyExistsException == false) { - logger.error("Failed to create the health node task", e); - if (enabled) { - clusterService.addListener(taskStarter); - } - } - }) - ); - } + } + }) + ); } } diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index a17bc885f6b65..6a45d1e5dc43e 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -83,7 +83,14 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl // check if the query has any parameters that are not in the supported set (if declared) Set supported = allSupportedParameters(); if (supported != null) { - var allSupported = Sets.union(RestResponse.RESPONSE_PARAMS, ALWAYS_SUPPORTED, supported); + var allSupported = Sets.union( + RestResponse.RESPONSE_PARAMS, + ALWAYS_SUPPORTED, + // these internal parameters cannot be set by end-users, but are used by Elasticsearch internally. + // they must be accepted by all handlers + RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, + supported + ); if (allSupported.containsAll(request.params().keySet()) == false) { Set unsupported = Sets.difference(request.params().keySet(), allSupported); throw new IllegalArgumentException(unrecognized(request, unsupported, allSupported, "parameter")); diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 8592888d2dd03..8e9cbd686110b 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -480,6 +480,14 @@ private void dispatchRequest( } else { threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.TRUE.toString()); } + + if (apiProtections.isEnabled()) { + // API protections are only enabled in serverless; therefore we can use this as an indicator to mark the + // request as a serverless mode request here, so downstream handlers can use the marker + request.markAsServerlessRequest(); + logger.trace("Marked request for uri [{}] as serverless request", request.uri()); + } + final var finalChannel = responseChannel; this.interceptor.intercept(request, responseChannel, handler.getConcreteRestHandler(), new ActionListener<>() { @Override diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index 66ba0c743813e..96f2c2d10dc96 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -48,7 +48,31 @@ public class RestRequest implements ToXContent.Params, Traceable { - public static final String PATH_RESTRICTED = "pathRestricted"; + /** + * Internal marker request parameter to indicate that a request was made in serverless mode. Use this parameter, together with + * {@link #OPERATOR_REQUEST} if you need to toggle behavior for serverless, for example to enforce partial API restrictions + * (prevent request fields, omit response fields) for an API. + * Requests not made in serverless mode, will *not* have this parameter set. + * Given a request instance, you can use {@link #isServerlessRequest()} to determine if the parameter is set or not. + * This is also available from {@code ToXContent.Params}. For example: + * {@code params.paramAsBoolean(RestRequest.SERVERLESS_REQUEST, false)} + */ + public static final String SERVERLESS_REQUEST = "serverlessRequest"; + /** + * Internal marker request parameter to indicate that a request was made by an operator user. + * Requests made by regular users (users without operator privileges), will *not* have this parameter set. + * Given a request instance, you can use {@link #isOperatorRequest()} to determine if the parameter is set or not. + * This is also available from {@code ToXContent.Params}. For example: + * {@code params.paramAsBoolean(RestRequest.OPERATOR_REQUEST, false)} + */ + public static final String OPERATOR_REQUEST = "operatorRequest"; + + /** + * Internal request parameters used as markers to indicate various operations modes such as serverless mode, or operator mode. + * These can never be set directly by end-users. Instead, they are set internally by Elasticsearch and must be supported by all + * request handlers. + */ + public static final Set INTERNAL_MARKER_REQUEST_PARAMETERS = Set.of(SERVERLESS_REQUEST, OPERATOR_REQUEST); // tchar pattern as defined by RFC7230 section 3.2.6 private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-Z0-9!#$%&'*+\\-.\\^_`|~]+"); @@ -616,13 +640,41 @@ public boolean hasExplicitRestApiVersion() { return restApiVersion.isPresent(); } - public void markPathRestricted(String restriction) { - if (params.containsKey(PATH_RESTRICTED)) { - throw new IllegalArgumentException("The parameter [" + PATH_RESTRICTED + "] is already defined."); + /** + * See {@link #SERVERLESS_REQUEST} + */ + public void markAsServerlessRequest() { + setParamTrueOnceAndConsume(SERVERLESS_REQUEST); + } + + /** + * See {@link #SERVERLESS_REQUEST} + */ + public boolean isServerlessRequest() { + return paramAsBoolean(SERVERLESS_REQUEST, false); + } + + /** + * See {@link #OPERATOR_REQUEST} + */ + public void markAsOperatorRequest() { + setParamTrueOnceAndConsume(OPERATOR_REQUEST); + } + + /** + * See {@link #OPERATOR_REQUEST} + */ + public boolean isOperatorRequest() { + return paramAsBoolean(OPERATOR_REQUEST, false); + } + + private void setParamTrueOnceAndConsume(String param) { + if (params.containsKey(param)) { + throw new IllegalArgumentException("The parameter [" + param + "] is already defined."); } - params.put(PATH_RESTRICTED, restriction); + params.put(param, "true"); // this parameter is intended be consumed via ToXContent.Params.param(..), not this.params(..) so don't require it is consumed here - consumedParams.add(PATH_RESTRICTED); + consumedParams.add(param); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/RestUtils.java b/server/src/main/java/org/elasticsearch/rest/RestUtils.java index 0e7200fa83b1c..681f4c33eb77c 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestUtils.java +++ b/server/src/main/java/org/elasticsearch/rest/RestUtils.java @@ -23,7 +23,7 @@ import java.util.regex.Pattern; import static org.elasticsearch.action.support.master.AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS; public class RestUtils { @@ -85,8 +85,10 @@ private static String decodeQueryStringParam(final String s) { } private static void addParam(Map params, String name, String value) { - if (PATH_RESTRICTED.equalsIgnoreCase(name)) { - throw new IllegalArgumentException("parameter [" + PATH_RESTRICTED + "] is reserved and may not set"); + for (var reservedParameter : INTERNAL_MARKER_REQUEST_PARAMETERS) { + if (reservedParameter.equalsIgnoreCase(name)) { + throw new IllegalArgumentException("parameter [" + name + "] is reserved and may not be set"); + } } params.put(name, value); } diff --git a/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java b/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java index 34aa04c5e484b..8a078db7dc012 100644 --- a/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java +++ b/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java @@ -22,9 +22,4 @@ @Target(ElementType.TYPE) public @interface ServerlessScope { Scope value(); - - /** - * A value used when restricting a response of a serverless endpoints. - */ - String SERVERLESS_RESTRICTION = "serverless"; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 6f70938a1e5e3..cff044a829bf0 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -278,8 +278,6 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final BigArrays bigArrays; - private final DfsPhase dfsPhase = new DfsPhase(); - private final FetchPhase fetchPhase; private final RankFeatureShardPhase rankFeatureShardPhase; private volatile boolean enableSearchWorkerThreads; @@ -511,7 +509,7 @@ private DfsSearchResult executeDfsPhase(ShardSearchRequest request, SearchShardT Releasable ignored = readerContext.markAsUsed(getKeepAlive(request)); SearchContext context = createContext(readerContext, request, task, ResultsType.DFS, false) ) { - dfsPhase.execute(context); + DfsPhase.execute(context); return context.dfsResult(); } catch (Exception e) { logger.trace("Dfs phase failed", e); diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java index 8c40a283844b4..ac150af50fbb0 100644 --- a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java @@ -52,7 +52,9 @@ */ public class DfsPhase { - public void execute(SearchContext context) { + private DfsPhase() {} + + public static void execute(SearchContext context) { try { collectStatistics(context); executeKnnVectorQuery(context); @@ -65,7 +67,7 @@ public void execute(SearchContext context) { } } - private void collectStatistics(SearchContext context) throws IOException { + private static void collectStatistics(SearchContext context) throws IOException { final DfsProfiler profiler = context.getProfilers() == null ? null : context.getProfilers().getDfsProfiler(); Map fieldStatistics = new HashMap<>(); diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index 8cd6fe9720039..88bf3232a2b17 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -127,3 +127,4 @@ 8.14.2,8636001 8.14.3,8636001 8.15.0,8702002 +8.15.1,8702002 diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index df0df2d05ba5b..f89bbb5712634 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -127,3 +127,4 @@ 8.14.2,8505000 8.14.3,8505000 8.15.0,8512000 +8.15.1,8512000 diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index 8d4017a756e48..3a066ab85147d 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -15,7 +15,9 @@ import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -25,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.IndexingPressure; @@ -39,6 +42,7 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.After; @@ -49,6 +53,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -85,7 +90,9 @@ class TestTransportSimulateBulkAction extends TransportSimulateBulkAction { new ActionFilters(Set.of()), new IndexingPressure(Settings.EMPTY), EmptySystemIndices.INSTANCE, - indicesService + indicesService, + NamedXContentRegistry.EMPTY, + new IndexSettingProviders(Set.of()) ); } } @@ -198,17 +205,23 @@ public void onFailure(Exception e) { public void testIndexDataWithValidation() throws IOException { /* - * This test makes sure that we validate mappings if we're indexing into an index that exists. It simulates 3 cases: - * (1) An indexing request to a nonexistent index (the index is not in the cluster state) - * (2) An indexing request to an index with non-strict mappings, or an index request that is valid with respect to the mappings + * This test makes sure that we validate mappings. It simulates 7 cases: + * (1) An indexing request to an index with non-strict mappings, or an index request that is valid with respect to the mappings * (the index is in the cluster state, but our mock indicesService.withTempIndexService() does not throw an exception) - * (3) An indexing request that is invalid with respect to the mappings (the index is in the cluster state, and our mock + * (2) An indexing request that is invalid with respect to the mappings (the index is in the cluster state, and our mock * indicesService.withTempIndexService() throws an exception) + * (3) An indexing request to a nonexistent index that matches a V1 template and is valid with respect to the mappings + * (4) An indexing request to a nonexistent index that matches a V1 template and is invalid with respect to the mappings + * (5) An indexing request to a nonexistent index that matches a V2 template and is valid with respect to the mappings + * (6) An indexing request to a nonexistent index that matches a V2 template and is invalid with respect to the mappings + * (7) An indexing request to a nonexistent index that matches no templates */ Task task = mock(Task.class); // unused BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null); int bulkItemCount = randomIntBetween(0, 200); Map indicesMap = new HashMap<>(); + Map v1Templates = new HashMap<>(); + Map v2Templates = new HashMap<>(); Metadata.Builder metadataBuilder = new Metadata.Builder(); Set indicesWithInvalidMappings = new HashSet<>(); for (int i = 0; i < bulkItemCount; i++) { @@ -221,23 +234,43 @@ public void testIndexDataWithValidation() throws IOException { bulkRequest.add(indexRequest); // Now we randomly decide what we're going to simulate with requests to this index: String indexName = indexRequest.index(); - switch (between(0, 2)) { + switch (between(0, 6)) { case 0 -> { - // Index does not exist, so we don't put it in the indicesMap + // Indices that have non-strict mappings, or we're sending valid requests for their mappings + indicesMap.put(indexName, newIndexMetadata(indexName)); } case 1 -> { - // Indices that have non-strict mappings, or we're sending valid requests for their mappings + // // Indices that we'll pretend to have sent invalid requests to + indicesWithInvalidMappings.add(indexName); indicesMap.put(indexName, newIndexMetadata(indexName)); } case 2 -> { - // Indices that we'll pretend to have sent invalid requests to + // Index does not exist, but matches a V1 template + v1Templates.put(indexName, newV1Template(indexName)); + } + case 3 -> { + // Index does not exist, but matches a V1 template + v1Templates.put(indexName, newV1Template(indexName)); indicesWithInvalidMappings.add(indexName); - indicesMap.put(indexName, newIndexMetadata(indexName)); + } + case 4 -> { + // Index does not exist, but matches a V2 template + v2Templates.put(indexName, newV2Template(indexName)); + } + case 5 -> { + // Index does not exist, but matches a V2 template + v2Templates.put(indexName, newV2Template(indexName)); + indicesWithInvalidMappings.add(indexName); + } + case 6 -> { + // Index does not exist, and matches no template } default -> throw new AssertionError("Illegal branch"); } } metadataBuilder.indices(indicesMap); + metadataBuilder.templates(v1Templates); + metadataBuilder.indexTemplates(v2Templates); ClusterServiceUtils.setState(clusterService, new ClusterState.Builder(clusterService.state()).metadata(metadataBuilder)); AtomicBoolean onResponseCalled = new AtomicBoolean(false); ActionListener listener = new ActionListener<>() { @@ -324,7 +357,7 @@ public void onFailure(Exception e) { fail(e, "Unexpected error"); } }; - when(indicesService.withTempIndexService(any(), any())).thenAnswer((Answer) invocation -> { + when(indicesService.withTempIndexService(any(), any())).thenAnswer((Answer) invocation -> { IndexMetadata imd = invocation.getArgument(0); if (indicesWithInvalidMappings.contains(imd.getIndex().getName())) { throw new ElasticsearchException("invalid mapping"); @@ -347,6 +380,14 @@ private IndexMetadata newIndexMetadata(String indexName) { return new IndexMetadata.Builder(indexName).settings(dummyIndexSettings).build(); } + private IndexTemplateMetadata newV1Template(String indexName) { + return new IndexTemplateMetadata.Builder(indexName).patterns(List.of(indexName)).build(); + } + + private ComposableIndexTemplate newV2Template(String indexName) { + return ComposableIndexTemplate.builder().indexPatterns(List.of(indexName)).build(); + } + private String convertMapToJsonString(Map map) throws IOException { try (XContentBuilder builder = JsonXContent.contentBuilder().map(map)) { return BytesReference.bytes(builder).utf8ToString(); diff --git a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java index bb06dbe5d09aa..ae88215f951de 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java @@ -31,7 +31,8 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.OPERATOR_REQUEST; +import static org.elasticsearch.rest.RestRequest.SERVERLESS_REQUEST; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -249,16 +250,30 @@ public void testRequiredContent() { assertEquals("unknown content type", e.getMessage()); } - public void testMarkPathRestricted() { + public void testIsServerlessRequest() { RestRequest request1 = contentRestRequest("content", new HashMap<>()); - request1.markPathRestricted("foo"); - assertEquals(request1.param(PATH_RESTRICTED), "foo"); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> request1.markPathRestricted("foo")); - assertThat(exception.getMessage(), is("The parameter [" + PATH_RESTRICTED + "] is already defined.")); - - RestRequest request2 = contentRestRequest("content", Map.of(PATH_RESTRICTED, "foo")); - exception = expectThrows(IllegalArgumentException.class, () -> request2.markPathRestricted("bar")); - assertThat(exception.getMessage(), is("The parameter [" + PATH_RESTRICTED + "] is already defined.")); + request1.markAsServerlessRequest(); + assertEquals(request1.param(SERVERLESS_REQUEST), "true"); + assertTrue(request1.isServerlessRequest()); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, request1::markAsServerlessRequest); + assertThat(exception.getMessage(), is("The parameter [" + SERVERLESS_REQUEST + "] is already defined.")); + + RestRequest request2 = contentRestRequest("content", Map.of(SERVERLESS_REQUEST, "true")); + exception = expectThrows(IllegalArgumentException.class, request2::markAsServerlessRequest); + assertThat(exception.getMessage(), is("The parameter [" + SERVERLESS_REQUEST + "] is already defined.")); + } + + public void testIsOperatorRequest() { + RestRequest request1 = contentRestRequest("content", new HashMap<>()); + request1.markAsOperatorRequest(); + assertEquals(request1.param(OPERATOR_REQUEST), "true"); + assertTrue(request1.isOperatorRequest()); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, request1::markAsOperatorRequest); + assertThat(exception.getMessage(), is("The parameter [" + OPERATOR_REQUEST + "] is already defined.")); + + RestRequest request2 = contentRestRequest("content", Map.of(OPERATOR_REQUEST, "true")); + exception = expectThrows(IllegalArgumentException.class, request2::markAsOperatorRequest); + assertThat(exception.getMessage(), is("The parameter [" + OPERATOR_REQUEST + "] is already defined.")); } public static RestRequest contentRestRequest(String content, Map params) { diff --git a/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java b/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java index 3226ca2bf51d2..24d40fd1b95fd 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java @@ -18,7 +18,7 @@ import java.util.Map; import java.util.regex.Pattern; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -160,13 +160,15 @@ public void testCrazyURL() { } public void testReservedParameters() { - Map params = new HashMap<>(); - String uri = "something?" + PATH_RESTRICTED + "=value"; - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params) - ); - assertEquals(exception.getMessage(), "parameter [" + PATH_RESTRICTED + "] is reserved and may not set"); + for (var reservedParam : INTERNAL_MARKER_REQUEST_PARAMETERS) { + Map params = new HashMap<>(); + String uri = "something?" + reservedParam + "=value"; + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params) + ); + assertEquals(exception.getMessage(), "parameter [" + reservedParam + "] is reserved and may not be set"); + } } private void assertCorsSettingRegexIsNull(String settingsValue) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 36d0240ed765b..6529d4d18fa5d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -394,12 +394,14 @@ static RoleDescriptor kibanaSystem(String name) { TransportUpdateSettingsAction.TYPE.name() ) .build(), - // For src/dest indices of the Cloud Security Posture packages that ships a + // For source indices of the Cloud Security Posture packages that ships a // transform RoleDescriptor.IndicesPrivileges.builder() .indices("logs-cloud_security_posture.findings-*", "logs-cloud_security_posture.vulnerabilities-*") .privileges("read", "view_index_metadata") .build(), + // For destination indices of the Cloud Security Posture packages that ships a + // transform RoleDescriptor.IndicesPrivileges.builder() .indices( "logs-cloud_security_posture.findings_latest-default*", @@ -415,17 +417,23 @@ static RoleDescriptor kibanaSystem(String name) { TransportUpdateSettingsAction.TYPE.name() ) .build(), + // For source indices of the Cloud Detection & Response (CDR) packages that ships a + // transform RoleDescriptor.IndicesPrivileges.builder() - .indices("logs-wiz.vulnerability-*") + .indices("logs-wiz.vulnerability-*", "logs-wiz.cloud_configuration_finding-*") .privileges("read", "view_index_metadata") .build(), + // For alias indices of the Cloud Detection & Response (CDR) packages that ships a + // transform RoleDescriptor.IndicesPrivileges.builder() // manage privilege required by the index alias - .indices("security_solution-*.vulnerability_latest") + .indices("security_solution-*.vulnerability_latest", "security_solution-*.misconfiguration_latest") .privileges("manage", TransportIndicesAliasesAction.NAME, TransportUpdateSettingsAction.TYPE.name()) .build(), + // For destination indices of the Cloud Detection & Response (CDR) packages that ships a + // transform RoleDescriptor.IndicesPrivileges.builder() - .indices("security_solution-*.vulnerability_latest-*") + .indices("security_solution-*.vulnerability_latest-*", "security_solution-*.misconfiguration_latest-*") .privileges( "create_index", "index", diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 258b2378b8a1c..be4042ae77838 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -1609,8 +1609,11 @@ public void testKibanaSystemRole() { assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(true)); }); - Arrays.asList("logs-wiz.vulnerability-" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((cspIndex) -> { - final IndexAbstraction indexAbstraction = mockIndexAbstraction(cspIndex); + Arrays.asList( + "logs-wiz.vulnerability-" + randomAlphaOfLength(randomIntBetween(0, 13)), + "logs-wiz.cloud_configuration_finding-" + randomAlphaOfLength(randomIntBetween(0, 13)) + ).forEach(indexName -> { + final IndexAbstraction indexAbstraction = mockIndexAbstraction(indexName); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false)); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false)); assertThat( @@ -1643,7 +1646,8 @@ public void testKibanaSystemRole() { "logs-cloud_security_posture.findings_latest-default-" + Version.CURRENT, "logs-cloud_security_posture.scores-default-" + Version.CURRENT, "logs-cloud_security_posture.vulnerabilities_latest-default" + Version.CURRENT, - "security_solution-*.vulnerability_latest-" + Version.CURRENT + "security_solution-*.vulnerability_latest-" + Version.CURRENT, + "security_solution-*.misconfiguration_latest-" + Version.CURRENT ).forEach(indexName -> { logger.info("index name [{}]", indexName); final IndexAbstraction indexAbstraction = mockIndexAbstraction(indexName); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java index 4ae860f394022..360936373a010 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java @@ -103,8 +103,7 @@ public void testPositionParserAtTokenAfterField_ThrowsWithMalformedJSON() throws XContentEOFException.class, () -> XContentUtils.positionParserAtTokenAfterField(parser, missingField, errorFormat) ); - - assertThat(exception.getMessage(), containsString("Unexpected end-of-input")); + assertThat(exception.getMessage(), containsString("[4:1] Unexpected end of file")); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java index e350a539ba928..e28d4f9608ae5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java @@ -309,8 +309,8 @@ public void testFails_ResponseIsInvalidJson_MissingSquareBracket() { new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ) ); - - assertThat(thrownException.getMessage(), containsString("expected close marker for Array (start marker at")); + assertThat(thrownException.getMessage(), containsString("[5:1] Unexpected end of file")); + assertThat(thrownException.getCause().getMessage(), containsString("expected close marker for Array (start marker at")); } public void testFails_ResponseIsInvalidJson_MissingField() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReader.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReader.java index 93f043bb5878b..ff55a50e46541 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReader.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReader.java @@ -8,7 +8,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.xcontent.XContentEOFException; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; @@ -196,7 +195,7 @@ protected void initArrays(String[] record, boolean[] gotFields) { protected XContentParser.Token tryNextTokenOrReadToEndOnError() throws IOException { try { return parser.nextToken(); - } catch (XContentEOFException | XContentParseException e) { + } catch (XContentParseException e) { logger.warn("Attempting to recover from malformed JSON data.", e); for (int i = 0; i <= nestedLevel; ++i) { readToEndOfObject(); @@ -217,7 +216,7 @@ protected void readToEndOfObject() throws IOException { do { try { token = parser.nextToken(); - } catch (XContentEOFException | XContentParseException e) { + } catch (XContentParseException e) { ++errorCounter; if (errorCounter >= PARSE_ERRORS_LIMIT) { logger.error("Failed to recover from malformed JSON data.", e); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java index f0889f1c48c75..ef3070f0bd787 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.security.operator; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.transport.TransportRequest; @@ -23,21 +22,10 @@ public interface OperatorOnlyRegistry { OperatorPrivilegesViolation check(String action, TransportRequest request); /** - * Checks to see if a given {@link RestHandler} is subject to operator-only restrictions for the REST API. - * - * Any REST API may be fully or partially restricted. - * A fully restricted REST API mandates that the implementation of this method throw an - * {@link org.elasticsearch.ElasticsearchStatusException} with an appropriate status code and error message. - * - * A partially restricted REST API mandates that the {@link RestRequest} is marked as restricted so that the downstream handler can - * behave appropriately. - * For example, to restrict the REST response the implementation - * should call {@link RestRequest#markPathRestricted(String)} so that the downstream handler can properly restrict the response - * before returning to the client. Note - a partial restriction should not throw an exception. - * - * @param restHandler The {@link RestHandler} to check for any restrictions - * @param restRequest The {@link RestRequest} to check for any restrictions and mark any partially restricted REST API's - * @throws ElasticsearchStatusException if the request should be denied in its entirety (fully restricted) + * This method is only called if the user is not an operator. + * Implementations should fail the request if the {@link RestRequest} is not allowed to proceed by throwing an + * {@link org.elasticsearch.ElasticsearchException}. If the request should be handled by the associated {@link RestHandler}, + * then this implementations should do nothing. */ void checkRest(RestHandler restHandler, RestRequest restRequest) throws ElasticsearchException; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java index 79c529eb3d7b1..9ef41fad12401 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java @@ -182,6 +182,9 @@ public boolean checkRest(RestHandler restHandler, RestRequest restRequest, RestC ); throw e; } + } else { + restRequest.markAsOperatorRequest(); + logger.trace("Marked request for uri [{}] as operator request", restRequest.uri()); } return true; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java index 5f0657079e694..e0ef46dc73a18 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java @@ -85,9 +85,9 @@ public RestResponse buildResponse(GetBuiltinPrivilegesResponse response, XConten @Override protected Exception innerCheckFeatureAvailable(RestRequest request) { - final boolean restrictPath = request.hasParam(RestRequest.PATH_RESTRICTED); - assert false == restrictPath || DiscoveryNode.isStateless(settings); - if (false == restrictPath) { + final boolean shouldRestrictForServerless = shouldRestrictForServerless(request); + assert false == shouldRestrictForServerless || DiscoveryNode.isStateless(settings); + if (false == shouldRestrictForServerless) { return super.innerCheckFeatureAvailable(request); } // This is a temporary hack: we are re-using the native roles setting as an overall feature flag for custom roles. @@ -106,4 +106,7 @@ protected Exception innerCheckFeatureAvailable(RestRequest request) { } } + private boolean shouldRestrictForServerless(RestRequest request) { + return request.isServerlessRequest() && false == request.isOperatorRequest(); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java index 232d74d16725d..dc9ecbbc63a8d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.security.rest.action.role; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.RestApiVersion; @@ -54,9 +53,9 @@ public String getName() { @Override public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { final String[] roles = request.paramAsStringArray("name", Strings.EMPTY_ARRAY); - final boolean restrictRequest = isPathRestricted(request); + final boolean restrictToNativeRolesOnly = request.isServerlessRequest() && false == request.isOperatorRequest(); return channel -> new GetRolesRequestBuilder(client).names(roles) - .nativeOnly(restrictRequest) + .nativeOnly(restrictToNativeRolesOnly) .execute(new RestBuilderListener<>(channel) { @Override public RestResponse buildResponse(GetRolesResponse response, XContentBuilder builder) throws Exception { @@ -84,17 +83,10 @@ protected Exception innerCheckFeatureAvailable(RestRequest request) { // Note: For non-restricted requests this action handles both reserved roles and native // roles, and should still be available even if native role management is disabled. // For restricted requests it should only be available if native role management is enabled - final boolean restrictPath = isPathRestricted(request); - if (false == restrictPath) { + if (false == request.isServerlessRequest() || request.isOperatorRequest()) { return null; } else { return super.innerCheckFeatureAvailable(request); } } - - private boolean isPathRestricted(RestRequest request) { - final boolean restrictRequest = request.hasParam(RestRequest.PATH_RESTRICTED); - assert false == restrictRequest || DiscoveryNode.isStateless(settings); - return restrictRequest; - } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java index aa95ea097413c..a5dabe8c2dd82 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java @@ -89,7 +89,7 @@ public void testWillNotCheckWhenLicenseDoesNotSupport() { verifyNoMoreInteractions(operatorOnlyRegistry); } - public void testMarkOperatorUser() throws IllegalAccessException { + public void testMarkOperatorUser() { final Settings settings = Settings.builder().put("xpack.security.operator_privileges.enabled", true).build(); when(xPackLicenseState.isAllowed(Security.OPERATOR_PRIVILEGES_FEATURE)).thenReturn(true); final User operatorUser = new User("operator_user"); @@ -204,7 +204,7 @@ public void testCheckWillPassForInternalUsersBecauseTheyHaveOperatorPrivileges() verify(operatorOnlyRegistry, never()).check(anyString(), any()); } - public void testMaybeInterceptRequest() throws IllegalAccessException { + public void testMaybeInterceptRequest() { final boolean licensed = randomBoolean(); when(xPackLicenseState.isAllowed(Security.OPERATOR_PRIVILEGES_FEATURE)).thenReturn(licensed); @@ -279,11 +279,16 @@ public void testCheckRest() { ); assertThat(ex, instanceOf(ElasticsearchSecurityException.class)); assertThat(ex, throwableWithMessage("violation!")); + verify(restRequest, never()).markAsOperatorRequest(); Mockito.clearInvocations(operatorOnlyRegistry); + Mockito.clearInvocations(restRequest); // is an operator threadContext.putHeader(AuthenticationField.PRIVILEGE_CATEGORY_KEY, AuthenticationField.PRIVILEGE_CATEGORY_VALUE_OPERATOR); verifyNoInteractions(operatorOnlyRegistry); assertTrue(operatorPrivilegesService.checkRest(restHandler, restRequest, restChannel, threadContext)); + verify(restRequest, times(1)).markAsOperatorRequest(); + Mockito.clearInvocations(operatorOnlyRegistry); + Mockito.clearInvocations(restRequest); } } diff --git a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java index c28e5f1e0fce8..cd17723c2635c 100644 --- a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java +++ b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java @@ -86,7 +86,8 @@ public abstract class KerberosTestCase extends ESTestCase { "sd", "mni", "sat", - "sa" + "sa", + "bgc" ); @BeforeClass