diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json index f529c8b51483c..62eb47821e0aa 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json @@ -30,6 +30,12 @@ ] }, "params":{ + "force_synthetic_source": { + "type": "boolean", + "description": "Should this request force synthetic _source? Use this to test if the mapping supports synthetic _source and to get a sense of the worst case performance. Fetches with this enabled will be slower the enabling synthetic source natively in the index.", + "visibility": "feature_flag", + "feature_flag": "es.index_mode_feature_flag_registered" + }, "stored_fields":{ "type":"list", "description":"A comma-separated list of stored fields to return in the response" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml index 4c10574714980..d2ba8750c9175 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml @@ -33,3 +33,94 @@ keyword: - match: _source: kwd: foo + +--- +force_synthetic_source_ok: + - skip: + version: " - 8.3.99" + reason: introduced in 8.4.0 + + - do: + indices.create: + index: test + body: + mappings: + _source: + synthetic: false + properties: + obj: + properties: + kwd: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + obj.kwd: foo + + # When _source is used in the fetch the original _source is perfect + - do: + get: + index: test + id: 1 + - match: + _source: + obj.kwd: foo + + # When we force synthetic source dots in field names get turned into objects + - do: + get: + index: test + id: 1 + force_synthetic_source: true + - match: + _source: + obj: + kwd: foo + +--- +force_synthetic_source_bad_mapping: + - skip: + version: " - 8.3.99" + reason: introduced in 8.4.0 + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 # Use a single shard to get consistent error messages + mappings: + _source: + synthetic: false + properties: + text: + type: text + + - do: + index: + index: test + id: 1 + refresh: true + body: + text: foo + + # When _source is used in the fetch the original _source is perfect + - do: + get: + index: test + id: 1 + - match: + _source: + text: foo + + # Forcing synthetic source fails because the mapping is invalid + - do: + catch: bad_request + get: + index: test + id: 1 + force_synthetic_source: true diff --git a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java index 23c412fe7795d..4f1a8d5f9d54d 100644 --- a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java +++ b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import java.io.IOException; @@ -55,6 +56,14 @@ public class GetRequest extends SingleShardRequest implements Realti private VersionType versionType = VersionType.INTERNAL; private long version = Versions.MATCH_ANY; + /** + * Should this request force {@link SourceLoader.Synthetic synthetic source}? + * Use this to test if the mapping supports synthetic _source and to get a sense + * of the worst case performance. Fetches with this enabled will be slower the + * enabling synthetic source natively in the index. + */ + private boolean forceSyntheticSource = false; + public GetRequest() {} GetRequest(StreamInput in) throws IOException { @@ -72,6 +81,11 @@ public GetRequest() {} this.versionType = VersionType.fromValue(in.readByte()); this.version = in.readLong(); fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::readFrom); + if (in.getVersion().onOrAfter(Version.V_8_4_0)) { + forceSyntheticSource = in.readBoolean(); + } else { + forceSyntheticSource = false; + } } @Override @@ -90,6 +104,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeByte(versionType.getValue()); out.writeLong(version); out.writeOptionalWriteable(fetchSourceContext); + if (out.getVersion().onOrAfter(Version.V_8_4_0)) { + out.writeBoolean(forceSyntheticSource); + } else { + if (forceSyntheticSource) { + throw new IllegalArgumentException("force_synthetic_source is not supported before 8.4.0"); + } + } } /** @@ -242,6 +263,26 @@ public VersionType versionType() { return this.versionType; } + /** + * Should this request force {@link SourceLoader.Synthetic synthetic source}? + * Use this to test if the mapping supports synthetic _source and to get a sense + * of the worst case performance. Fetches with this enabled will be slower the + * enabling synthetic source natively in the index. + */ + public void setForceSyntheticSource(boolean forceSyntheticSource) { + this.forceSyntheticSource = forceSyntheticSource; + } + + /** + * Should this request force {@link SourceLoader.Synthetic synthetic source}? + * Use this to test if the mapping supports synthetic _source and to get a sense + * of the worst case performance. Fetches with this enabled will be slower the + * enabling synthetic source natively in the index. + */ + public boolean isForceSyntheticSource() { + return forceSyntheticSource; + } + @Override public String toString() { return "get [" + index + "][" + id + "]: routing [" + routing + "]"; diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index ad1b4793c8c1b..182fdf087a8c2 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -116,7 +116,8 @@ protected GetResponse shardOperation(GetRequest request, ShardId shardId) throws request.realtime(), request.version(), request.versionType(), - request.fetchSourceContext() + request.fetchSourceContext(), + request.isForceSyntheticSource() ); return new GetResponse(result); } diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 6a3134c4781ea..d9713b66fec14 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -117,7 +117,15 @@ protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, Sha MultiGetRequest.Item item = request.items.get(i); try { GetResult getResult = indexShard.getService() - .get(item.id(), item.storedFields(), request.realtime(), item.version(), item.versionType(), item.fetchSourceContext()); + .get( + item.id(), + item.storedFields(), + request.realtime(), + item.version(), + item.versionType(), + item.fetchSourceContext(), + false + ); response.add(request.locations.get(i), new GetResponse(getResult)); } catch (RuntimeException e) { if (TransportActions.isShardNotAvailableException(e)) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 3493b46733a5b..0ec66549e43b2 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -755,6 +755,9 @@ public boolean isForceSyntheticSource() { /** * Should this request force {@link SourceLoader.Synthetic synthetic source}? + * Use this to test if the mapping supports synthetic _source and to get a sense + * of the worst case performance. Fetches with this enabled will be slower the + * enabling synthetic source natively in the index. */ public void setForceSyntheticSource(boolean forceSyntheticSource) { this.forceSyntheticSource = forceSyntheticSource; diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index b3e67530a8e49..683db3c0d8c33 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -70,9 +71,20 @@ public GetResult get( boolean realtime, long version, VersionType versionType, - FetchSourceContext fetchSourceContext + FetchSourceContext fetchSourceContext, + boolean forceSyntheticSource ) throws IOException { - return get(id, gFields, realtime, version, versionType, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, fetchSourceContext); + return get( + id, + gFields, + realtime, + version, + versionType, + UNASSIGNED_SEQ_NO, + UNASSIGNED_PRIMARY_TERM, + fetchSourceContext, + forceSyntheticSource + ); } private GetResult get( @@ -83,12 +95,23 @@ private GetResult get( VersionType versionType, long ifSeqNo, long ifPrimaryTerm, - FetchSourceContext fetchSourceContext + FetchSourceContext fetchSourceContext, + boolean forceSyntheticSource ) throws IOException { currentMetric.inc(); try { long now = System.nanoTime(); - GetResult getResult = innerGet(id, gFields, realtime, version, versionType, ifSeqNo, ifPrimaryTerm, fetchSourceContext); + GetResult getResult = innerGet( + id, + gFields, + realtime, + version, + versionType, + ifSeqNo, + ifPrimaryTerm, + fetchSourceContext, + forceSyntheticSource + ); if (getResult.isExists()) { existsMetric.inc(System.nanoTime() - now); @@ -110,7 +133,8 @@ public GetResult getForUpdate(String id, long ifSeqNo, long ifPrimaryTerm) throw VersionType.INTERNAL, ifSeqNo, ifPrimaryTerm, - FetchSourceContext.FETCH_SOURCE + FetchSourceContext.FETCH_SOURCE, + false ); } @@ -131,7 +155,7 @@ public GetResult get(Engine.GetResult engineGetResult, String id, String[] field try { long now = System.nanoTime(); fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, fields); - GetResult getResult = innerGetLoadFromStoredFields(id, fields, fetchSourceContext, engineGetResult); + GetResult getResult = innerGetFetch(id, fields, fetchSourceContext, engineGetResult, false); if (getResult.isExists()) { existsMetric.inc(System.nanoTime() - now); } else { @@ -169,7 +193,8 @@ private GetResult innerGet( VersionType versionType, long ifSeqNo, long ifPrimaryTerm, - FetchSourceContext fetchSourceContext + FetchSourceContext fetchSourceContext, + boolean forceSyntheticSource ) throws IOException { fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, gFields); @@ -189,17 +214,18 @@ private GetResult innerGet( try { // break between having loaded it from translog (so we only have _source), and having a document to load - return innerGetLoadFromStoredFields(id, gFields, fetchSourceContext, get); + return innerGetFetch(id, gFields, fetchSourceContext, get, forceSyntheticSource); } finally { get.close(); } } - private GetResult innerGetLoadFromStoredFields( + private GetResult innerGetFetch( String id, String[] storedFields, FetchSourceContext fetchSourceContext, - Engine.GetResult get + Engine.GetResult get, + boolean forceSyntheticSource ) throws IOException { assert get.exists() : "method should only be called if document could be retrieved"; @@ -228,7 +254,10 @@ private GetResult innerGetLoadFromStoredFields( } catch (IOException e) { throw new ElasticsearchException("Failed to get id [" + id + "]", e); } - source = mappingLookup.newSourceLoader().leaf(docIdAndVersion.reader).source(fieldVisitor, docIdAndVersion.docId); + SourceLoader loader = forceSyntheticSource + ? new SourceLoader.Synthetic(mappingLookup.getMapping()) + : mappingLookup.newSourceLoader(); + source = loader.leaf(docIdAndVersion.reader).source(fieldVisitor, docIdAndVersion.docId); // put stored fields into result objects if (fieldVisitor.fields().isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java index 8ce6fe71dfe2d..c89910f1b6349 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -78,6 +79,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC getRequest.versionType(VersionType.fromString(request.param("version_type"), getRequest.versionType())); getRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request)); + if (IndexSettings.isTimeSeriesModeEnabled() && request.paramAsBoolean("force_synthetic_source", false)) { + getRequest.setForceSyntheticSource(true); + } return channel -> client.get(getRequest, new RestToXContentListener(channel) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index b9aa0d33704c5..4bb0124e828ec 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -160,7 +160,7 @@ private void runGetFromTranslogWithOptions( Engine.IndexResult test2 = indexDoc(primary, "2", docToIndex, XContentType.JSON, "foobar"); assertTrue(primary.getEngine().refreshNeeded()); GetResult testGet2 = primary.getService() - .get("2", new String[] { "foo" }, true, 1, VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE); + .get("2", new String[] { "foo" }, true, 1, VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE, false); assertEquals(new String(testGet2.source() == null ? new byte[0] : testGet2.source(), StandardCharsets.UTF_8), expectedResult); assertTrue(testGet2.getFields().containsKey(RoutingFieldMapper.NAME)); assertTrue(testGet2.getFields().containsKey("foo")); @@ -174,7 +174,8 @@ private void runGetFromTranslogWithOptions( assertEquals(searcher.getIndexReader().maxDoc(), 3); } - testGet2 = primary.getService().get("2", new String[] { "foo" }, true, 1, VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE); + testGet2 = primary.getService() + .get("2", new String[] { "foo" }, true, 1, VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE, false); assertEquals(new String(testGet2.source() == null ? new byte[0] : testGet2.source(), StandardCharsets.UTF_8), expectedResult); assertTrue(testGet2.getFields().containsKey(RoutingFieldMapper.NAME)); assertTrue(testGet2.getFields().containsKey("foo"));