Add a cluster setting to disallow loading fielddata on _id field (#49166

)

This change adds a dynamic cluster setting named `indices.id_field_data.enabled`.
When set to `false` any attempt to load the fielddata for the `_id` field will fail
with an exception. The default value in this change is set to `false` in order to prevent
fielddata usage on this field for future versions but it will be set to `true` when backporting
to 7x. When the setting is set to true (manually or by default in 7x) the loading will also issue
a deprecation warning since we want to disallow fielddata entirely when #26472
is implemented.

Closes #43599

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -116,32 +116,32 @@ public void indexDocuments() throws IOException {
         {
             Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/type/1");
             doc1.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
-            doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}");
+            doc1.setJsonEntity("{\"type\":\"type1\", \"id\":1, \"num\":10, \"num2\":50}");
             client().performRequest(doc1);
             Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/type/2");
             doc2.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
-            doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}");
+            doc2.setJsonEntity("{\"type\":\"type1\", \"id\":2, \"num\":20, \"num2\":40}");
             client().performRequest(doc2);
             Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/type/3");
             doc3.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
-            doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}");
+            doc3.setJsonEntity("{\"type\":\"type1\", \"id\":3, \"num\":50, \"num2\":35}");
             client().performRequest(doc3);
             Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/type/4");
             doc4.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
-            doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
+            doc4.setJsonEntity("{\"type\":\"type2\", \"id\":4, \"num\":100, \"num2\":10}");
             client().performRequest(doc4);
             Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/type/5");
             doc5.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
-            doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
+            doc5.setJsonEntity("{\"type\":\"type2\", \"id\":5, \"num\":100, \"num2\":10}");
             client().performRequest(doc5);
         }
 
         {
             Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1");
-            doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}");
+            doc1.setJsonEntity("{\"id\":1, \"field\":\"value1\", \"rating\": 7}");
             client().performRequest(doc1);
             Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/2");
-            doc2.setJsonEntity("{\"field\":\"value2\"}");
+            doc2.setJsonEntity("{\"id\":2, \"field\":\"value2\"}");
             client().performRequest(doc2);
         }
 
@@ -159,19 +159,19 @@ public void indexDocuments() throws IOException {
                 "}");
             client().performRequest(create);
             Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3");
-            doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}");
+            doc3.setJsonEntity("{\"id\":3, \"field\":\"value1\", \"rating\": \"good\"}");
             client().performRequest(doc3);
             Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4");
-            doc4.setJsonEntity("{\"field\":\"value2\"}");
+            doc4.setJsonEntity("{\"id\":4, \"field\":\"value2\"}");
             client().performRequest(doc4);
         }
 
         {
             Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5");
-            doc5.setJsonEntity("{\"field\":\"value1\"}");
+            doc5.setJsonEntity("{\"id\":5, \"field\":\"value1\"}");
             client().performRequest(doc5);
             Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/6");
-            doc6.setJsonEntity("{\"field\":\"value2\"}");
+            doc6.setJsonEntity("{\"id\":6, \"field\":\"value2\"}");
             client().performRequest(doc6);
         }
 
@@ -194,7 +194,7 @@ public void indexDocuments() throws IOException {
                     "}");
             client().performRequest(create);
             Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/_doc/1");
-            doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}");
+            doc1.setJsonEntity("{\"id\":1, \"field1\":\"value1\", \"field2\":\"value2\"}");
             client().performRequest(doc1);
 
             Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases");
@@ -232,7 +232,7 @@ public void testSearchNoQuery() throws IOException {
             assertEquals(1.0f, searchHit.getScore(), 0);
             assertEquals(-1L, searchHit.getVersion());
             assertNotNull(searchHit.getSourceAsMap());
-            assertEquals(3, searchHit.getSourceAsMap().size());
+            assertEquals(4, searchHit.getSourceAsMap().size());
             assertTrue(searchHit.getSourceAsMap().containsKey("type"));
             assertTrue(searchHit.getSourceAsMap().containsKey("num"));
             assertTrue(searchHit.getSourceAsMap().containsKey("num2"));
@@ -257,7 +257,7 @@ public void testSearchMatchQuery() throws IOException {
         assertThat(searchHit.getScore(), greaterThan(0f));
         assertEquals(-1L, searchHit.getVersion());
         assertNotNull(searchHit.getSourceAsMap());
-        assertEquals(3, searchHit.getSourceAsMap().size());
+        assertEquals(4, searchHit.getSourceAsMap().size());
         assertEquals("type1", searchHit.getSourceAsMap().get("type"));
         assertEquals(50, searchHit.getSourceAsMap().get("num2"));
     }
@@ -713,13 +713,13 @@ public void testSearchScroll() throws Exception {
     public void testMultiSearch() throws Exception {
         MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
         SearchRequest searchRequest1 = new SearchRequest("index1");
-        searchRequest1.source().sort("_id", SortOrder.ASC);
+        searchRequest1.source().sort("id", SortOrder.ASC);
         multiSearchRequest.add(searchRequest1);
         SearchRequest searchRequest2 = new SearchRequest("index2");
-        searchRequest2.source().sort("_id", SortOrder.ASC);
+        searchRequest2.source().sort("id", SortOrder.ASC);
         multiSearchRequest.add(searchRequest2);
         SearchRequest searchRequest3 = new SearchRequest("index3");
-        searchRequest3.source().sort("_id", SortOrder.ASC);
+        searchRequest3.source().sort("id", SortOrder.ASC);
         multiSearchRequest.add(searchRequest3);
 
         MultiSearchResponse multiSearchResponse =
@@ -1211,7 +1211,8 @@ public void testExplainWithFetchSource() throws IOException {
             assertTrue(explainResponse.hasExplanation());
             assertThat(explainResponse.getExplanation().getValue(), equalTo(1.0f));
             assertTrue(explainResponse.getGetResult().isExists());
-            assertThat(explainResponse.getGetResult().getSource(), equalTo(Collections.singletonMap("field1", "value1")));
+            assertEquals(2, explainResponse.getGetResult().getSource().size());
+            assertThat(explainResponse.getGetResult().getSource().get("field1"), equalTo("value1"));
         }
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java

@@ -169,7 +169,7 @@ public void testSearch() throws Exception {
 
             // tag::search-source-sorting
             sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1>
-            sourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC));  // <2>
+            sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC));  // <2>
             // end::search-source-sorting
 
             // tag::search-source-filtering-off
@@ -1251,6 +1251,9 @@ private void indexSearchTestData() throws IOException {
         CreateIndexRequest authorsRequest = new CreateIndexRequest("authors")
             .mapping(XContentFactory.jsonBuilder().startObject()
                 .startObject("properties")
+                    .startObject("id")
+                        .field("type", "keyword")
+                    .endObject()
                     .startObject("user")
                         .field("type", "keyword")
                         .field("doc_values", "false")
@@ -1263,6 +1266,9 @@ private void indexSearchTestData() throws IOException {
         CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors")
             .mapping(XContentFactory.jsonBuilder().startObject()
                 .startObject("properties")
+                    .startObject("id")
+                        .field("type", "keyword")
+                    .endObject()
                     .startObject("user")
                         .field("type", "keyword")
                         .field("store", "true")
@@ -1274,19 +1280,19 @@ private void indexSearchTestData() throws IOException {
 
         BulkRequest bulkRequest = new BulkRequest();
         bulkRequest.add(new IndexRequest("posts").id("1")
-                .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
+                .source(XContentType.JSON, "id", 1, "title", "In which order are my Elasticsearch queries executed?", "user",
                         Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
         bulkRequest.add(new IndexRequest("posts").id("2")
-                .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
+                .source(XContentType.JSON, "id", 2, "title", "Current status and upcoming changes in Elasticsearch", "user",
                         Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
         bulkRequest.add(new IndexRequest("posts").id("3")
-                .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
+                .source(XContentType.JSON, "id", 3, "title", "The Future of Federated Search in Elasticsearch", "user",
                         Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
 
         bulkRequest.add(new IndexRequest("authors").id("1")
-            .source(XContentType.JSON, "user", "kimchy"));
+            .source(XContentType.JSON, "id", 1, "user", "kimchy"));
         bulkRequest.add(new IndexRequest("contributors").id("1")
-            .source(XContentType.JSON, "user", "tanguy"));
+            .source(XContentType.JSON, "id", 1, "user", "tanguy"));
 
 
         bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

docs/reference/mapping/types/parent-join.asciidoc

@@ -16,6 +16,9 @@ PUT my_index
 {
   "mappings": {
     "properties": {
+      "my_id": {
+        "type": "keyword"
+      },
       "my_join_field": { <1>
         "type": "join",
         "relations": {
@@ -38,6 +41,7 @@ For instance the following example creates two `parent` documents in the `questi
 --------------------------------------------------
 PUT my_index/_doc/1?refresh
 {
+  "my_id": "1",
   "text": "This is a question",
   "my_join_field": {
     "name": "question" <1>
@@ -46,6 +50,7 @@ PUT my_index/_doc/1?refresh
 
 PUT my_index/_doc/2?refresh
 {
+  "my_id": "2",
   "text": "This is another question",
   "my_join_field": {
     "name": "question"
@@ -63,12 +68,14 @@ as a shortcut instead of encapsulating it in the normal object notation:
 --------------------------------------------------
 PUT my_index/_doc/1?refresh
 {
+  "my_id": "1",
   "text": "This is a question",
   "my_join_field": "question" <1>
 }
 
 PUT my_index/_doc/2?refresh
 {
+  "my_id": "2",
   "text": "This is another question",
   "my_join_field": "question"
 }
@@ -89,6 +96,7 @@ For instance the following example shows how to index two `child` documents:
 --------------------------------------------------
 PUT my_index/_doc/3?routing=1&refresh <1>
 {
+  "my_id": "3",
   "text": "This is an answer",
   "my_join_field": {
     "name": "answer", <2>
@@ -98,6 +106,7 @@ PUT my_index/_doc/3?routing=1&refresh <1>
 
 PUT my_index/_doc/4?routing=1&refresh
 {
+  "my_id": "4",
   "text": "This is another answer",
   "my_join_field": {
     "name": "answer",
@@ -159,7 +168,7 @@ GET my_index/_search
   "query": {
     "match_all": {}
   },
-  "sort": ["_id"]
+  "sort": ["my_id"]
 }
 --------------------------
 // TEST[continued]
@@ -183,6 +192,7 @@ Will return:
                 "_id": "1",
                 "_score": null,
                 "_source": {
+                    "my_id": "1",
                     "text": "This is a question",
                     "my_join_field": "question" <1>
                 },
@@ -196,6 +206,7 @@ Will return:
                 "_id": "2",
                 "_score": null,
                 "_source": {
+                    "my_id": "2",
                     "text": "This is another question",
                     "my_join_field": "question" <2>
                 },
@@ -210,6 +221,7 @@ Will return:
                 "_score": null,
                 "_routing": "1",
                 "_source": {
+                    "my_id": "3",
                     "text": "This is an answer",
                     "my_join_field": {
                         "name": "answer", <3>
@@ -227,6 +239,7 @@ Will return:
                 "_score": null,
                 "_routing": "1",
                 "_source": {
+                    "my_id": "4",
                     "text": "This is another answer",
                     "my_join_field": {
                         "name": "answer",

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java

@@ -79,8 +79,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) {
 
         SearchRequestBuilder req = client().prepareSearch().setIndices("test");
         req.setQuery(QueryBuilders.matchAllQuery())
-                .addSort(SortBuilders.fieldSort("_id")
-                        .order(SortOrder.ASC))
+                .addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC).unmappedType("long"))
                 .addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap));
         return req;
     }
@@ -147,8 +146,10 @@ public void testDateMethods() throws Exception {
         ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
         ensureGreen("test");
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
-                client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
+                client().prepareIndex("test", "doc", "1")
+                    .setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
+                client().prepareIndex("test", "doc", "2")
+                    .setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
         SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get();
         assertEquals(2, rsp.getHits().getTotalHits().value);
         SearchHits hits = rsp.getHits();
@@ -175,8 +176,10 @@ public void testDateObjectMethods() throws Exception {
         ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
         ensureGreen("test");
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
-                client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
+                client().prepareIndex("test", "doc", "1")
+                    .setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
+                client().prepareIndex("test", "doc", "2")
+                    .setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
         SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get();
         assertEquals(2, rsp.getHits().getTotalHits().value);
         SearchHits hits = rsp.getHits();
@@ -207,15 +210,18 @@ public void testMultiValueMethods() throws Exception {
         ensureGreen("test");
 
         Map<String, Object> doc1 = new HashMap<>();
+        doc1.put("id", 1);
         doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d});
         doc1.put("double1", new Double[]{1.2d, 2.4d});
         doc1.put("double2", 3.0d);
 
         Map<String, Object> doc2 = new HashMap<>();
+        doc2.put("id", 2);
         doc2.put("double0", 5.0d);
         doc2.put("double1", 3.0d);
 
         Map<String, Object> doc3 = new HashMap<>();
+        doc3.put("id", 3);
         doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d});
         doc3.put("double1", 4.0d);
 
@@ -319,8 +325,8 @@ public void testSparseField() throws Exception {
         ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long"));
         ensureGreen("test");
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("x", 4),
-                client().prepareIndex("test", "doc", "2").setSource("y", 2));
+                client().prepareIndex("test", "doc", "1").setSource("id", 1, "x", 4),
+                client().prepareIndex("test", "doc","2").setSource("id", 2, "y", 2));
         SearchResponse rsp = buildRequest("doc['x'] + 1").get();
         ElasticsearchAssertions.assertSearchResponse(rsp);
         SearchHits hits = rsp.getHits();
@@ -348,9 +354,9 @@ public void testParams() throws Exception {
         createIndex("test");
         ensureGreen("test");
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("x", 10),
-                client().prepareIndex("test", "doc", "2").setSource("x", 3),
-                client().prepareIndex("test", "doc", "3").setSource("x", 5));
+                client().prepareIndex("test", "doc", "1").setSource("id", 1, "x", 10),
+                client().prepareIndex("test", "doc", "2").setSource("id", 2, "x", 3),
+                client().prepareIndex("test", "doc", "3").setSource("id", 3, "x", 5));
         // a = int, b = double, c = long
         String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
         SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get();
@@ -622,9 +628,9 @@ public void testBoolean() throws Exception {
         assertAcked(prepareCreate("test").addMapping("doc", xContentBuilder));
         ensureGreen();
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("price", 1.0, "vip", true),
-                client().prepareIndex("test", "doc", "2").setSource("price", 2.0, "vip", false),
-                client().prepareIndex("test", "doc", "3").setSource("price", 2.0, "vip", false));
+                client().prepareIndex("test", "doc", "1").setSource("id", 1, "price", 1.0, "vip", true),
+                client().prepareIndex("test", "doc", "2").setSource("id", 2, "price", 2.0, "vip", false),
+                client().prepareIndex("test", "doc", "3").setSource("id", 3, "price", 2.0, "vip", false));
         // access .value
         SearchResponse rsp = buildRequest("doc['vip'].value").get();
         assertSearchResponse(rsp);
@@ -653,8 +659,8 @@ public void testFilterScript() throws Exception {
         createIndex("test");
         ensureGreen("test");
         indexRandom(true,
-            client().prepareIndex("test", "doc", "1").setSource("foo", 1.0),
-            client().prepareIndex("test", "doc", "2").setSource("foo", 0.0));
+            client().prepareIndex("test", "doc", "1").setSource("id", 1, "foo", 1.0),
+            client().prepareIndex("test", "doc", "2").setSource("id", 2, "foo", 0.0));
         SearchRequestBuilder builder = buildRequest("doc['foo'].value");
         Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap());
         builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script)));