diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
index 0d8bf59823acf..2e167812615e1 100644
--- a/.idea/runConfigurations/Debug_OpenSearch.xml
+++ b/.idea/runConfigurations/Debug_OpenSearch.xml
@@ -8,4 +8,4 @@
-
+
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e43865ad12b31..385b89f7417e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530))
- Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
- Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708))
+- Add fetch phase profiling. ([#18664](https://github.com/opensearch-project/OpenSearch/pull/18664))
- Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697))
- Add the configurable limit on rule cardinality ([#18663](https://github.com/opensearch-project/OpenSearch/pull/18663))
- [Experimental] Start in "clusterless" mode if a clusterless ClusterPlugin is loaded ([#18479](https://github.com/opensearch-project/OpenSearch/pull/18479))
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_script_fields_profile.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_script_fields_profile.yml
new file mode 100644
index 0000000000000..beaf597dd768c
--- /dev/null
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_script_fields_profile.yml
@@ -0,0 +1,69 @@
+setup:
+ - do:
+ indices.create:
+ index: test_fetch_profile
+ body:
+ settings:
+ number_of_replicas: 0
+ number_of_shards: 1
+ mappings:
+ properties:
+ text_field:
+ type: text
+ fields:
+ keyword:
+ type: keyword
+ numeric_field:
+ type: integer
+ date_field:
+ type: date
+ object_field:
+ type: nested
+ properties:
+ nested_field:
+ type: keyword
+ stored_field:
+ type: keyword
+ store: true
+
+ - do:
+ bulk:
+ refresh: true
+ index: test_fetch_profile
+ body: |
+ { "index": {} }
+ { "text_field": "Hello world", "numeric_field": 42, "date_field": "2023-01-01", "object_field": { "nested_field": "nested value" }, "stored_field": "stored value" }
+ { "index": {} }
+ { "text_field": "Another document", "numeric_field": 100, "date_field": "2023-02-01", "object_field": { "nested_field": "another nested" }, "stored_field": "another stored" }
+ { "index": {} }
+ { "text_field": "Third document with more text", "numeric_field": 200, "date_field": "2023-03-01", "object_field": { "nested_field": "third nested" }, "stored_field": "third stored" }
+
+
+---
+"Script fields phase profiling":
+ - skip:
+ features: "contains"
+
+ - do:
+ search:
+ index: test_fetch_profile
+ body:
+ profile: true
+ query:
+ match_all: {}
+ script_fields:
+ my_field:
+ script:
+ lang: painless
+ source: "doc['numeric_field'].value * 2"
+
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "ScriptFieldsPhase"
+ description: "ScriptFieldsPhase"
+ - length: { profile.shards.0.fetch.0.children: 1 }
+
+ - is_true: profile.shards.0.fetch.0.children.0.breakdown.process
+ - match: { profile.shards.0.fetch.0.children.0.breakdown.process_count: 3 }
+ - is_true: profile.shards.0.fetch.0.children.0.breakdown.set_next_reader
+ - match: { profile.shards.0.fetch.0.children.0.breakdown.set_next_reader_count: 1 }
diff --git a/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java
index d3460c616f454..a81062a584277 100644
--- a/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java
+++ b/qa/multi-cluster-search/src/test/java/org/opensearch/search/CCSDuelIT.java
@@ -831,10 +831,12 @@ private static Map responseToMap(SearchResponse response) throws
Map responseMap = org.opensearch.common.xcontent.XContentHelper.convertToMap(bytesReference, false, MediaTypeRegistry.JSON).v2();
assertNotNull(responseMap.put("took", -1));
responseMap.remove("num_reduce_phases");
- Map profile = (Map)responseMap.get("profile");
+ Map profile = (Map) responseMap.get("profile");
if (profile != null) {
- List> shards = (List >)profile.get("shards");
+ List> shards = (List>) profile.get("shards");
for (Map shard : shards) {
+ // Unconditionally remove the fetch profile to prevent flaky failures
+ shard.remove("fetch");
replaceProfileTime(shard);
}
}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.profile/10_fetch_phase.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.profile/10_fetch_phase.yml
new file mode 100644
index 0000000000000..1de34ee7e3984
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.profile/10_fetch_phase.yml
@@ -0,0 +1,193 @@
+# Setup remains the same
+setup:
+ - do:
+ indices.create:
+ index: test_fetch_profile
+ body:
+ settings:
+ number_of_replicas: 0
+ number_of_shards: 1
+ mappings:
+ properties:
+ text_field:
+ type: text
+ fields:
+ keyword:
+ type: keyword
+ numeric_field:
+ type: integer
+ date_field:
+ type: date
+ object_field:
+ type: nested
+ properties:
+ nested_field:
+ type: keyword
+ stored_field:
+ type: keyword
+ store: true
+
+ - do:
+ bulk:
+ refresh: true
+ index: test_fetch_profile
+ body: |
+ { "index": {} }
+ { "text_field": "Hello world", "numeric_field": 42, "date_field": "2023-01-01", "object_field": { "nested_field": "nested value" }, "stored_field": "stored value" }
+ { "index": {} }
+ { "text_field": "Another document", "numeric_field": 100, "date_field": "2023-02-01", "object_field": { "nested_field": "another nested" }, "stored_field": "another stored" }
+ { "index": {} }
+ { "text_field": "Third document with more text", "numeric_field": 200, "date_field": "2023-03-01", "object_field": { "nested_field": "third nested" }, "stored_field": "third stored" }
+
+---
+"Combined fetch sub-phases profiling":
+ - skip:
+ version: " - 3.1.99"
+ reason: "Fetch phase profiling was introduced in 3.2.0"
+ features: "contains"
+
+ - do:
+ search:
+ index: test_fetch_profile
+ body:
+ profile: true
+ explain: true
+ version: true
+ seq_no_primary_term: true
+ track_scores: true
+ sort:
+ - numeric_field: { order: asc }
+ query:
+ bool:
+ must:
+ - match_all: {}
+ should:
+ - nested:
+ path: "object_field"
+ query:
+ wildcard:
+ "object_field.nested_field": "nested*"
+ inner_hits: {}
+ - match:
+ text_field:
+ query: "document"
+ _name: "my_query"
+ docvalue_fields:
+ - "numeric_field"
+ fields:
+ - "stored_field"
+ highlight:
+ fields:
+ "object_field.nested_field": {}
+
+ # 1. Verify basic fetch profile structure
+ - is_true: profile.shards.0.fetch.0
+ - match: { profile.shards.0.fetch.0.type: "fetch" }
+ - match: { profile.shards.0.fetch.0.description: "fetch" }
+ - is_true: profile.shards.0.fetch.0.time_in_nanos
+
+ # 2. Verify detailed breakdown of the main fetch operation
+ - is_true: profile.shards.0.fetch.0.breakdown
+ - is_true: profile.shards.0.fetch.0.breakdown.load_stored_fields
+ - match: { profile.shards.0.fetch.0.breakdown.load_stored_fields_count: 3 }
+ - is_true: profile.shards.0.fetch.0.breakdown.load_source
+ - match: { profile.shards.0.fetch.0.breakdown.load_source_count: 3 }
+ - is_true: profile.shards.0.fetch.0.breakdown.get_next_reader
+ - match: { profile.shards.0.fetch.0.breakdown.get_next_reader_count: 1}
+ - is_true: profile.shards.0.fetch.0.breakdown.build_sub_phase_processors
+ - match: { profile.shards.0.fetch.0.breakdown.build_sub_phase_processors_count: 1}
+ - is_true: profile.shards.0.fetch.0.breakdown.create_stored_fields_visitor
+ - match: { profile.shards.0.fetch.0.breakdown.create_stored_fields_visitor_count: 1}
+
+ # 3. Verify all expected fetch sub-phases are present as children
+ - length: { profile.shards.0.fetch.0.children: 9 }
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "FetchSourcePhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "ExplainPhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "FetchDocValuesPhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "FetchFieldsPhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "FetchVersionPhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "SeqNoPrimaryTermPhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "MatchedQueriesPhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "HighlightPhase"
+ - contains:
+ profile.shards.0.fetch.0.children:
+ type: "FetchScorePhase"
+
+---
+"No source or empty fetch profiling":
+ - skip:
+ version: " - 3.1.99"
+ reason: "Fetch phase profiling was introduced in 3.2.0"
+
+ # Case 1: Test with _source: false, which removes FetchSourcePhase
+ - do:
+ search:
+ index: test_fetch_profile
+ body:
+ profile: true
+ query:
+ match_all: {}
+ _source: false
+ docvalue_fields:
+ - "numeric_field"
+
+ - is_true: profile.shards.0.fetch.0
+ - length: { profile.shards.0.fetch.0.children: 1 }
+ - match: { profile.shards.0.fetch.0.children.0.type: "FetchDocValuesPhase" }
+ - match: { profile.shards.0.fetch.0.children.0.breakdown.process_count: 3 }
+
+ # Case 2: Test with size: 0, which results in an empty fetch profile
+ - do:
+ search:
+ index: test_fetch_profile
+ body:
+ profile: true
+ size: 0
+ aggs:
+ group_by_nested:
+ terms:
+ field: "object_field.nested_field"
+
+ - match: { profile.shards.0.fetch: [ ] }
+
+---
+"Top-hits aggregation profiling":
+ - skip:
+ version: " - 3.1.99"
+ reason: "Fetch phase profiling was introduced in 3.2.0"
+
+ - do:
+ search:
+ index: test_fetch_profile
+ body:
+ profile: true
+ query:
+ match:
+ text_field: "document"
+ aggs:
+ top_hits_agg:
+ top_hits:
+ size: 1
+
+ # Verify that the profile contains a single fetch operation for the query
+ - length: { profile.shards.0.fetch: 1 }
+ - match: { profile.shards.0.fetch.0.type: "fetch" }
+ - match: { profile.shards.0.fetch.0.description: "fetch" }
+ - length: { profile.shards.0.fetch.0.children: 1 }
+ - match: { profile.shards.0.fetch.0.children.0.type: "FetchSourcePhase" }
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/profile/fetch/FetchProfilerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/profile/fetch/FetchProfilerIT.java
new file mode 100644
index 0000000000000..d85d6bb08b211
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/search/profile/fetch/FetchProfilerIT.java
@@ -0,0 +1,390 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.fetch;
+
+import org.apache.lucene.tests.util.English;
+import org.opensearch.action.index.IndexRequestBuilder;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.search.SearchType;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.index.query.InnerHitBuilder;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.opensearch.search.profile.ProfileResult;
+import org.opensearch.search.profile.ProfileShardResult;
+import org.opensearch.search.sort.SortOrder;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class FetchProfilerIT extends OpenSearchIntegTestCase {
+
+ @Override
+ protected int numberOfShards() {
+ return 1; // Use a single shard to ensure all documents are in one shard
+ }
+
+ /**
+ * This test verifies that the fetch profiler returns reasonable results for a simple match_all query
+ */
+ public void testRootProfile() throws Exception {
+ createIndex("test");
+
+ int numDocs = randomIntBetween(100, 150);
+ IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i);
+ }
+
+ indexRandom(true, docs);
+ ensureGreen();
+
+ QueryBuilder q = QueryBuilders.matchAllQuery();
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(q)
+ .setProfile(true)
+ .setSize(numDocs)
+ .setSearchType(SearchType.QUERY_THEN_FETCH)
+ .get();
+
+ Map profileResults = resp.getProfileResults();
+ assertNotNull(profileResults);
+ assertFalse("Profile response should not be an empty array", profileResults.isEmpty());
+
+ for (Map.Entry shardResult : profileResults.entrySet()) {
+ FetchProfileShardResult fetchProfileResult = shardResult.getValue().getFetchProfileResult();
+ assertNotNull("Fetch profile result should not be null", fetchProfileResult);
+
+ List fetchProfileResults = fetchProfileResult.getFetchProfileResults();
+ assertNotNull("Fetch profile results should not be null", fetchProfileResults);
+ assertFalse("Should have at least one fetch profile result", fetchProfileResults.isEmpty());
+
+ for (ProfileResult fetchResult : fetchProfileResults) {
+ Map breakdown = fetchResult.getTimeBreakdown();
+ assertNotNull("Time breakdown should not be null", breakdown);
+
+ assertTrue(
+ "CREATE_STORED_FIELDS_VISITOR timing should be present",
+ breakdown.containsKey(FetchTimingType.CREATE_STORED_FIELDS_VISITOR.toString())
+ );
+ assertTrue(
+ "BUILD_SUB_PHASE_PROCESSORS timing should be present",
+ breakdown.containsKey(FetchTimingType.BUILD_SUB_PHASE_PROCESSORS.toString())
+ );
+ assertTrue("GET_NEXT_READER timing should be present", breakdown.containsKey(FetchTimingType.GET_NEXT_READER.toString()));
+ assertTrue(
+ "LOAD_STORED_FIELDS timing should be present",
+ breakdown.containsKey(FetchTimingType.LOAD_STORED_FIELDS.toString())
+ );
+ assertTrue("LOAD_SOURCE timing should be present", breakdown.containsKey(FetchTimingType.LOAD_SOURCE.toString()));
+ }
+ }
+ assertFetchPhase(resp, "FetchSourcePhase", 1);
+ }
+
+ public void testExplainProfile() throws Exception {
+ createIndex("test");
+
+ int numDocs = randomIntBetween(10, 20);
+ IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i);
+ }
+
+ indexRandom(true, docs);
+ ensureGreen();
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setProfile(true)
+ .setExplain(true)
+ .setSize(numDocs)
+ .setSearchType(SearchType.QUERY_THEN_FETCH)
+ .get();
+
+ assertNotNull("Response should include explanations", resp.getHits().getAt(0).getExplanation());
+ assertFetchPhase(resp, "ExplainPhase", 2);
+ }
+
+ public void testDocValuesPhaseProfile() throws Exception {
+ createIndex("test");
+ int numDocs = randomIntBetween(10, 20);
+ IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i);
+ }
+ indexRandom(true, docs);
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setProfile(true)
+ .addDocValueField("field2")
+ .setSize(numDocs)
+ .get();
+
+ assertNotNull(resp.getHits().getAt(0).field("field2"));
+ assertFetchPhase(resp, "FetchDocValuesPhase", 2);
+ }
+
+ public void testFieldsPhaseProfile() throws Exception {
+ client().admin()
+ .indices()
+ .prepareCreate("test")
+ .setMapping(
+ XContentFactory.jsonBuilder()
+ .startObject()
+ .startObject("properties")
+ .startObject("field1")
+ .field("type", "text")
+ .field("store", true)
+ .endObject()
+ .startObject("field2")
+ .field("type", "integer")
+ .endObject()
+ .endObject()
+ .endObject()
+ )
+ .get();
+ ensureGreen("test");
+
+ int numDocs = randomIntBetween(10, 20);
+ IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i);
+ }
+ indexRandom(true, docs);
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setProfile(true)
+ .addFetchField("field1")
+ .setSize(numDocs)
+ .get();
+
+ assertNotNull(resp.getHits().getAt(0).field("field1"));
+ assertFetchPhase(resp, "FetchFieldsPhase", 2);
+ }
+
+ public void testVersionPhaseProfile() throws Exception {
+ createIndex("test");
+ int numDocs = randomIntBetween(10, 20);
+ IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i));
+ }
+ indexRandom(true, docs);
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setProfile(true)
+ .setVersion(true)
+ .setSize(numDocs)
+ .get();
+
+ assertEquals(1L, resp.getHits().getAt(0).getVersion());
+ assertFetchPhase(resp, "FetchVersionPhase", 2);
+ }
+
+ public void testSeqNoPrimaryTermPhaseProfile() throws Exception {
+ createIndex("test");
+ int numDocs = randomIntBetween(10, 20);
+ IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i));
+ }
+ indexRandom(true, docs);
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setProfile(true)
+ .seqNoAndPrimaryTerm(true)
+ .setSize(numDocs)
+ .get();
+
+ assertTrue(resp.getHits().getAt(0).getSeqNo() > -1L);
+ assertTrue(resp.getHits().getAt(0).getPrimaryTerm() > 0L);
+ assertFetchPhase(resp, "SeqNoPrimaryTermPhase", 2);
+ }
+
+ public void testMatchedQueriesPhaseProfile() throws Exception {
+ createIndex("test");
+ client().prepareIndex("test").setId("1").setSource("field1", "The quick brown fox").get();
+ refresh();
+
+ QueryBuilder q = QueryBuilders.boolQuery()
+ .should(QueryBuilders.termQuery("field1", "quick").queryName("first_query"))
+ .should(QueryBuilders.termQuery("field1", "fox").queryName("second_query"));
+
+ SearchResponse resp = client().prepareSearch("test").setQuery(q).setProfile(true).get();
+
+ List matchedQueries = Arrays.asList(resp.getHits().getAt(0).getMatchedQueries());
+ assertTrue(matchedQueries.contains("first_query"));
+ assertTrue(matchedQueries.contains("second_query"));
+ assertFetchPhase(resp, "MatchedQueriesPhase", 2);
+ }
+
+ public void testHighlightPhaseProfile() throws Exception {
+ createIndex("test");
+ client().prepareIndex("test").setId("1").setSource("field1", "The quick brown fox jumps over the lazy dog").get();
+ refresh();
+
+ QueryBuilder q = QueryBuilders.matchQuery("field1", "quick fox");
+ HighlightBuilder highlighter = new HighlightBuilder().field("field1");
+
+ SearchResponse resp = client().prepareSearch("test").setQuery(q).setProfile(true).highlighter(highlighter).get();
+
+ assertNotNull(resp.getHits().getAt(0).getHighlightFields().get("field1"));
+ assertFetchPhase(resp, "HighlightPhase", 2);
+ }
+
+ public void testFetchScorePhaseProfile() throws Exception {
+ createIndex("test");
+ client().prepareIndex("test").setId("1").setSource("field1", "The quick brown fox", "field2", 42).get();
+ refresh();
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(QueryBuilders.matchQuery("field1", "quick"))
+ .setProfile(true)
+ .setTrackScores(true)
+ .addSort("field2", SortOrder.ASC) // Sort by a field other than _score
+ .get();
+
+ assertTrue(resp.getHits().getAt(0).getScore() > 0.0f);
+
+ assertFetchPhase(resp, "FetchScorePhase", 2);
+ }
+
+ public void testInnerHitsPhaseProfile() throws Exception {
+ client().admin()
+ .indices()
+ .prepareCreate("test")
+ .setMapping(
+ XContentFactory.jsonBuilder()
+ .startObject()
+ .startObject("properties")
+ .startObject("nested_field")
+ .field("type", "nested")
+ .startObject("properties")
+ .startObject("nested_text")
+ .field("type", "text")
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject()
+ )
+ .get();
+ ensureGreen("test");
+
+ // Index a document with nested fields
+ client().prepareIndex("test")
+ .setId("1")
+ .setSource(
+ XContentFactory.jsonBuilder()
+ .startObject()
+ .startArray("nested_field")
+ .startObject()
+ .field("nested_text", "first nested value")
+ .endObject()
+ .startObject()
+ .field("nested_text", "second nested value")
+ .endObject()
+ .endArray()
+ .endObject()
+ )
+ .get();
+ refresh();
+
+ SearchResponse resp = client().prepareSearch("test")
+ .setQuery(
+ QueryBuilders.nestedQuery("nested_field", QueryBuilders.matchAllQuery(), org.apache.lucene.search.join.ScoreMode.None)
+ .innerHit(new InnerHitBuilder())
+ )
+ .setProfile(true)
+ .get();
+
+ assertFalse(resp.getHits().getAt(0).getInnerHits().isEmpty());
+
+ assertFetchPhase(resp, "FetchSourcePhase", 1);
+
+ // InnerHitsPhase should no longer be profiled
+
+ Map profileResults = resp.getProfileResults();
+
+ boolean foundInnerHitsPhase = false;
+ boolean foundFetchInnerHits = false;
+
+ for (ProfileShardResult shardResult : profileResults.values()) {
+ FetchProfileShardResult fetchProfileResult = shardResult.getFetchProfileResult();
+
+ for (ProfileResult fetchResult : fetchProfileResult.getFetchProfileResults()) {
+ for (ProfileResult phase : fetchResult.getProfiledChildren()) {
+ if ("InnerHitsPhase".equals(phase.getQueryName())) {
+
+ foundInnerHitsPhase = true;
+ Map breakdown = phase.getTimeBreakdown();
+ assertTrue(breakdown.containsKey(FetchTimingType.PROCESS.toString()));
+ assertTrue(breakdown.containsKey(FetchTimingType.SET_NEXT_READER.toString()));
+ }
+ }
+ if ("fetch_inner_hits".equals(fetchResult.getQueryName())) {
+ foundFetchInnerHits = true;
+ }
+ }
+ }
+ assertFalse("InnerHitsPhase should be absent", foundInnerHitsPhase);
+ assertFalse("fetch_inner_hits profile should be absent", foundFetchInnerHits);
+ }
+
+ private void assertFetchPhase(SearchResponse resp, String phaseName, int expectedChildren) {
+ Map profileResults = resp.getProfileResults();
+ assertNotNull(profileResults);
+ assertFalse(profileResults.isEmpty());
+
+ boolean foundPhase = false;
+ for (ProfileShardResult shardResult : profileResults.values()) {
+ FetchProfileShardResult fetchProfileResult = shardResult.getFetchProfileResult();
+ assertNotNull(fetchProfileResult);
+
+ for (ProfileResult fetchResult : fetchProfileResult.getFetchProfileResults()) {
+ for (ProfileResult child : fetchResult.getProfiledChildren()) {
+ assertEquals(
+ "Should have " + expectedChildren + " profiled children",
+ expectedChildren,
+ fetchResult.getProfiledChildren().size()
+ );
+ if (phaseName.equals(child.getQueryName())) {
+ Map breakdown = child.getTimeBreakdown();
+ assertTrue(
+ phaseName + " should have PROCESS timing type",
+ breakdown.containsKey(FetchTimingType.PROCESS.toString())
+ );
+ assertTrue(
+ phaseName + " should have NEXT_READER timing type",
+ breakdown.containsKey(FetchTimingType.SET_NEXT_READER.toString())
+ );
+ foundPhase = true;
+ break;
+ }
+ }
+ if (foundPhase) {
+ break;
+ }
+ }
+ if (foundPhase) {
+ break;
+ }
+ }
+ assertTrue(phaseName + " should be present in the profile", foundPhase);
+ }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java
index ba47e0f49ed2d..c41446da84bbb 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java
@@ -322,7 +322,6 @@ public void testBool() throws Exception {
assertThat(result.getTime(), greaterThan(0L));
}
}
-
}
/**
@@ -725,5 +724,4 @@ private void assertQueryProfileResult(ProfileResult result) {
assertThat(breakdown.size(), equalTo(27));
}
}
-
}
diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java
index 6baf3b514198e..df334d6f4ec76 100644
--- a/server/src/main/java/org/opensearch/search/SearchService.java
+++ b/server/src/main/java/org/opensearch/search/SearchService.java
@@ -131,7 +131,9 @@
import org.opensearch.search.internal.ShardSearchRequest;
import org.opensearch.search.lookup.SearchLookup;
import org.opensearch.search.profile.ProfileMetric;
+import org.opensearch.search.profile.ProfileShardResult;
import org.opensearch.search.profile.Profilers;
+import org.opensearch.search.profile.SearchProfileShardResults;
import org.opensearch.search.query.QueryPhase;
import org.opensearch.search.query.QuerySearchRequest;
import org.opensearch.search.query.QuerySearchResult;
@@ -766,6 +768,10 @@ private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchCon
try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, afterQueryTime)) {
shortcutDocIdsToLoad(context);
fetchPhase.execute(context);
+ if (context.getProfilers() != null) {
+ ProfileShardResult shardResults = SearchProfileShardResults.buildShardResults(context.getProfilers(), context.request());
+ context.queryResult().profileResults(shardResults);
+ }
if (reader.singleSession()) {
freeReaderContext(reader.id());
}
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java
index d7fec4a6c6aa9..e30cc9e1b3a4a 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/TopHitsAggregator.java
@@ -215,7 +215,7 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE
docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
}
subSearchContext.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
- fetchPhase.execute(subSearchContext);
+ fetchPhase.execute(subSearchContext, "fetch_top_hits_aggregation");
FetchSearchResult fetchResult = subSearchContext.fetchResult();
SearchHit[] internalHits = fetchResult.fetchResult().hits().getHits();
for (int i = 0; i < internalHits.length; i++) {
diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java
index df37b7dbfda98..244479fbd10e1 100644
--- a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java
@@ -41,6 +41,7 @@
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.BitSet;
import org.opensearch.common.CheckedBiConsumer;
+import org.opensearch.common.CheckedSupplier;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.document.DocumentField;
@@ -49,7 +50,6 @@
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.support.XContentMapValues;
-import org.opensearch.core.common.text.Text;
import org.opensearch.core.tasks.TaskCancelledException;
import org.opensearch.core.xcontent.MediaType;
import org.opensearch.index.fieldvisitor.CustomFieldsVisitor;
@@ -70,6 +70,10 @@
import org.opensearch.search.internal.SearchContext;
import org.opensearch.search.lookup.SearchLookup;
import org.opensearch.search.lookup.SourceLookup;
+import org.opensearch.search.profile.Timer;
+import org.opensearch.search.profile.fetch.FetchProfileBreakdown;
+import org.opensearch.search.profile.fetch.FetchProfiler;
+import org.opensearch.search.profile.fetch.FetchTimingType;
import java.io.IOException;
import java.util.ArrayList;
@@ -103,6 +107,20 @@ public FetchPhase(List fetchSubPhases) {
}
public void execute(SearchContext context) {
+ execute(context, "fetch");
+ }
+
+ public void execute(SearchContext context, String profileDescription) {
+ FetchProfileBreakdown breakdown = null;
+ FetchProfiler fetchProfiler = null;
+ if (context.getProfilers() != null && "fetch".equals(profileDescription)) { // second condition makes sure only standard fetch phase
+ // is profiled for now
+ fetchProfiler = context.getProfilers().getFetchProfiler();
+ if (context.docIdsToLoadSize() > 0) {
+ breakdown = fetchProfiler.startFetchPhase(profileDescription);
+ }
+ }
+
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("{}", new SearchContextSourcePrinter(context));
}
@@ -126,13 +144,33 @@ public void execute(SearchContext context) {
Arrays.sort(docs);
Map> storedToRequestedFields = new HashMap<>();
- FieldsVisitor fieldsVisitor = createStoredFieldsVisitor(context, storedToRequestedFields);
+ FieldsVisitor fieldsVisitor = profile(
+ breakdown,
+ FetchTimingType.CREATE_STORED_FIELDS_VISITOR,
+ () -> createStoredFieldsVisitor(context, storedToRequestedFields)
+ );
FetchContext fetchContext = new FetchContext(context);
SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()];
- List processors = getProcessors(context.shardTarget(), fetchContext);
+ List> processors = profile(
+ breakdown,
+ FetchTimingType.BUILD_SUB_PHASE_PROCESSORS,
+ () -> getProcessors(context.shardTarget(), fetchContext)
+ );
+
+ Map processorProfiles = new HashMap<>();
+ if (breakdown != null) {
+ for (Tuple p : processors) {
+ if (p.v2() instanceof InnerHitsPhase == false) {
+ FetchProfileBreakdown pb = context.getProfilers()
+ .getFetchProfiler()
+ .startSubPhase(p.v2().getClass().getSimpleName(), profileDescription);
+ processorProfiles.put(p.v1(), pb);
+ }
+ }
+ }
int currentReaderIndex = -1;
LeafReaderContext currentReaderContext = null;
@@ -146,7 +184,11 @@ public void execute(SearchContext context) {
try {
int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves());
if (currentReaderIndex != readerIndex) {
- currentReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex);
+ currentReaderContext = profile(
+ breakdown,
+ FetchTimingType.GET_NEXT_READER,
+ () -> context.searcher().getIndexReader().leaves().get(readerIndex)
+ );
currentReaderIndex = readerIndex;
if (currentReaderContext.reader() instanceof SequentialStoredFieldsLeafReader
&& hasSequentialDocs
@@ -160,8 +202,13 @@ public void execute(SearchContext context) {
} else {
fieldReader = currentReaderContext.reader().storedFields()::document;
}
- for (FetchSubPhaseProcessor processor : processors) {
- processor.setNextReader(currentReaderContext);
+ for (Tuple p : processors) {
+ FetchProfileBreakdown pbd = processorProfiles.get(p.v1());
+ LeafReaderContext readerCtx = currentReaderContext;
+ profile(pbd, FetchTimingType.SET_NEXT_READER, () -> {
+ p.v1().setNextReader(readerCtx);
+ return null;
+ });
}
}
assert currentReaderContext != null;
@@ -172,10 +219,16 @@ public void execute(SearchContext context) {
docId,
storedToRequestedFields,
currentReaderContext,
- fieldReader
+ fieldReader,
+ breakdown
);
- for (FetchSubPhaseProcessor processor : processors) {
- processor.process(hit);
+
+ for (Tuple p : processors) {
+ FetchProfileBreakdown pbd = processorProfiles.get(p.v1());
+ profile(pbd, FetchTimingType.PROCESS, () -> {
+ p.v1().process(hit);
+ return null;
+ });
}
hits[docs[index].index] = hit.hit();
} catch (Exception e) {
@@ -189,15 +242,18 @@ public void execute(SearchContext context) {
TotalHits totalHits = context.queryResult().getTotalHits();
context.fetchResult().hits(new SearchHits(hits, totalHits, context.queryResult().getMaxScore()));
+ if (fetchProfiler != null) {
+ fetchProfiler.endFetchPhase(profileDescription);
+ }
}
- List getProcessors(SearchShardTarget target, FetchContext context) {
+ List> getProcessors(SearchShardTarget target, FetchContext context) {
try {
- List processors = new ArrayList<>();
+ List> processors = new ArrayList<>();
for (FetchSubPhase fsp : fetchSubPhases) {
FetchSubPhaseProcessor processor = fsp.getProcessor(context);
if (processor != null) {
- processors.add(processor);
+ processors.add(new Tuple<>(processor, fsp));
}
}
return processors;
@@ -303,7 +359,8 @@ private HitContext prepareHitContext(
int docId,
Map> storedToRequestedFields,
LeafReaderContext subReaderContext,
- CheckedBiConsumer storedFieldReader
+ CheckedBiConsumer storedFieldReader,
+ FetchProfileBreakdown breakdown
) throws IOException {
int rootDocId = findRootDocumentIfNested(context, subReaderContext, docId - subReaderContext.docBase);
if (rootDocId == -1) {
@@ -314,10 +371,19 @@ private HitContext prepareHitContext(
docId,
storedToRequestedFields,
subReaderContext,
- storedFieldReader
+ storedFieldReader,
+ breakdown
);
} else {
- return prepareNestedHitContext(context, docId, rootDocId, storedToRequestedFields, subReaderContext, storedFieldReader);
+ return prepareNestedHitContext(
+ context,
+ docId,
+ rootDocId,
+ storedToRequestedFields,
+ subReaderContext,
+ storedFieldReader,
+ breakdown
+ );
}
}
@@ -335,18 +401,21 @@ private HitContext prepareNonNestedHitContext(
int docId,
Map> storedToRequestedFields,
LeafReaderContext subReaderContext,
- CheckedBiConsumer fieldReader
+ CheckedBiConsumer fieldReader,
+ FetchProfileBreakdown breakdown
) throws IOException {
int subDocId = docId - subReaderContext.docBase;
- DocumentMapper documentMapper = context.mapperService().documentMapper();
- Text typeText = documentMapper.typeText();
if (fieldsVisitor == null) {
SearchHit hit = new SearchHit(docId, null, null, null);
return new HitContext(hit, subReaderContext, subDocId, lookup.source());
} else {
SearchHit hit;
- loadStoredFields(context::fieldType, fieldReader, fieldsVisitor, subDocId);
+ profile(breakdown, FetchTimingType.LOAD_STORED_FIELDS, () -> {
+ loadStoredFields(context::fieldType, fieldReader, fieldsVisitor, subDocId);
+ return null;
+ });
+
String id = fieldsVisitor.id();
if (fieldsVisitor.fields().isEmpty() == false) {
Map docFields = new HashMap<>();
@@ -359,7 +428,10 @@ private HitContext prepareNonNestedHitContext(
HitContext hitContext = new HitContext(hit, subReaderContext, subDocId, lookup.source());
if (fieldsVisitor.source() != null) {
- hitContext.sourceLookup().setSource(fieldsVisitor.source());
+ profile(breakdown, FetchTimingType.LOAD_SOURCE, () -> {
+ hitContext.sourceLookup().setSource(fieldsVisitor.source());
+ return null;
+ });
}
return hitContext;
}
@@ -380,7 +452,8 @@ private HitContext prepareNestedHitContext(
int rootDocId,
Map> storedToRequestedFields,
LeafReaderContext subReaderContext,
- CheckedBiConsumer storedFieldReader
+ CheckedBiConsumer storedFieldReader,
+ FetchProfileBreakdown breakdown
) throws IOException {
// Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
@@ -399,12 +472,15 @@ private HitContext prepareNestedHitContext(
if (needSource) {
SourceLookup rootLookup = innerHitsContext.getRootLookup();
- rootSourceAsMap = rootLookup.loadSourceIfNeeded();
+ rootSourceAsMap = profile(breakdown, FetchTimingType.LOAD_SOURCE, rootLookup::loadSourceIfNeeded);
rootSourceContentType = rootLookup.sourceContentType();
}
} else {
FieldsVisitor rootFieldsVisitor = new FieldsVisitor(needSource);
- loadStoredFields(context::fieldType, storedFieldReader, rootFieldsVisitor, rootDocId);
+ profile(breakdown, FetchTimingType.LOAD_STORED_FIELDS, () -> {
+ loadStoredFields(context::fieldType, storedFieldReader, rootFieldsVisitor, rootDocId);
+ return null;
+ });
rootFieldsVisitor.postProcess(context::fieldType);
rootId = rootFieldsVisitor.id();
@@ -423,7 +499,11 @@ private HitContext prepareNestedHitContext(
Map metaFields = emptyMap();
if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), false);
- loadStoredFields(context::fieldType, storedFieldReader, nestedFieldsVisitor, nestedDocId);
+ profile(breakdown, FetchTimingType.LOAD_STORED_FIELDS, () -> {
+ loadStoredFields(context::fieldType, storedFieldReader, nestedFieldsVisitor, nestedDocId);
+ return null;
+ });
+
if (nestedFieldsVisitor.fields().isEmpty() == false) {
docFields = new HashMap<>();
metaFields = new HashMap<>();
@@ -602,6 +682,23 @@ private static void fillDocAndMetaFields(
}
}
+ private static T profile(
+ FetchProfileBreakdown breakdown,
+ FetchTimingType timingType,
+ CheckedSupplier supplier
+ ) throws E {
+ if (breakdown == null) {
+ return supplier.get();
+ }
+ Timer timer = breakdown.getTimer(timingType);
+ timer.start();
+ try {
+ return supplier.get();
+ } finally {
+ timer.stop();
+ }
+ }
+
/**
* Returns true if the provided docs are
* stored sequentially (Dn = Dn-1 + 1).
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
index 5855a0b3217f3..50dda07263695 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
@@ -50,6 +50,7 @@
import org.opensearch.search.internal.SearchContext;
import org.opensearch.search.internal.SubSearchContext;
import org.opensearch.search.lookup.SourceLookup;
+import org.opensearch.search.profile.Profilers;
import java.io.IOException;
import java.util.Arrays;
@@ -166,6 +167,11 @@ public SourceLookup getRootLookup() {
public void setRootLookup(SourceLookup rootLookup) {
this.rootLookup = rootLookup;
}
+
+ @Override
+ public Profilers getProfilers() {
+ return context.getProfilers();
+ }
}
public static void intersect(Weight weight, Weight innerHitQueryWeight, Collector collector, LeafReaderContext ctx) throws IOException {
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java
index 0b07dc35f13bb..af31be5f4ae93 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java
@@ -64,7 +64,7 @@ public InnerHitsPhase(FetchPhase fetchPhase) {
@Override
public FetchSubPhaseProcessor getProcessor(FetchContext searchContext) {
- if (searchContext.innerHits() == null) {
+ if (searchContext.innerHits() == null || searchContext.innerHits().getInnerHits().isEmpty()) {
return null;
}
Map innerHits = searchContext.innerHits().getInnerHits();
@@ -102,7 +102,7 @@ private void hitExecute(Map innerHi
innerHitsContext.setId(hit.getId());
innerHitsContext.setRootLookup(rootLookup);
- fetchPhase.execute(innerHitsContext);
+ fetchPhase.execute(innerHitsContext, "fetch_inner_hits");
FetchSearchResult fetchResult = innerHitsContext.fetchResult();
SearchHit[] internalHits = fetchResult.fetchResult().hits().getHits();
for (int j = 0; j < internalHits.length; j++) {
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java
index 67d1863050a7b..a31cbeb844c4f 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java
@@ -54,7 +54,7 @@ public final class ScriptFieldsPhase implements FetchSubPhase {
@Override
public FetchSubPhaseProcessor getProcessor(FetchContext context) {
- if (context.scriptFields() == null) {
+ if (context.scriptFields() == null || context.scriptFields().fields().isEmpty()) {
return null;
}
List scriptFields = context.scriptFields().fields();
diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java b/server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java
index 6924b85b9da8b..a6df7f4b78d07 100644
--- a/server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java
+++ b/server/src/main/java/org/opensearch/search/profile/ProfileMetricUtil.java
@@ -9,6 +9,7 @@
package org.opensearch.search.profile;
import org.opensearch.search.profile.aggregation.AggregationTimingType;
+import org.opensearch.search.profile.fetch.FetchTimingType;
import org.opensearch.search.profile.query.QueryTimingType;
import java.util.ArrayList;
@@ -41,4 +42,12 @@ public static Collection> getAggregationProfileMetrics()
}
return metrics;
}
+
+ public static Collection> getFetchProfileMetrics() {
+ Collection> metrics = new ArrayList<>();
+ for (FetchTimingType type : FetchTimingType.values()) {
+ metrics.add(() -> new Timer(type.toString()));
+ }
+ return metrics;
+ }
}
diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java
index 8ff622152ee70..14019716c9d3a 100644
--- a/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java
+++ b/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java
@@ -32,11 +32,13 @@
package org.opensearch.search.profile;
+import org.opensearch.Version;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.opensearch.search.profile.fetch.FetchProfileShardResult;
import org.opensearch.search.profile.query.QueryProfileShardResult;
import java.io.IOException;
@@ -56,18 +58,35 @@ public class ProfileShardResult implements Writeable {
private final AggregationProfileShardResult aggProfileShardResult;
+ private final FetchProfileShardResult fetchProfileResult;
+
private NetworkTime networkTime;
public ProfileShardResult(
List queryProfileResults,
AggregationProfileShardResult aggProfileShardResult,
+ FetchProfileShardResult fetchProfileResult,
NetworkTime networkTime
) {
this.aggProfileShardResult = aggProfileShardResult;
+ this.fetchProfileResult = fetchProfileResult;
this.queryProfileResults = Collections.unmodifiableList(queryProfileResults);
this.networkTime = networkTime;
}
+ /**
+ * Constructor for backwards compatibility.
+ * @deprecated Use {@link #ProfileShardResult(List, AggregationProfileShardResult, FetchProfileShardResult, NetworkTime)} instead
+ */
+ @Deprecated
+ public ProfileShardResult(
+ List queryProfileResults,
+ AggregationProfileShardResult aggProfileShardResult,
+ NetworkTime networkTime
+ ) {
+ this(queryProfileResults, aggProfileShardResult, new FetchProfileShardResult(Collections.emptyList()), networkTime);
+ }
+
public ProfileShardResult(StreamInput in) throws IOException {
int profileSize = in.readVInt();
List queryProfileResults = new ArrayList<>(profileSize);
@@ -77,6 +96,11 @@ public ProfileShardResult(StreamInput in) throws IOException {
}
this.queryProfileResults = Collections.unmodifiableList(queryProfileResults);
this.aggProfileShardResult = new AggregationProfileShardResult(in);
+ if (in.getVersion().onOrAfter(Version.V_3_2_0)) {
+ this.fetchProfileResult = new FetchProfileShardResult(in);
+ } else {
+ this.fetchProfileResult = new FetchProfileShardResult(Collections.emptyList());
+ }
this.networkTime = new NetworkTime(in);
}
@@ -87,6 +111,9 @@ public void writeTo(StreamOutput out) throws IOException {
queryShardResult.writeTo(out);
}
aggProfileShardResult.writeTo(out);
+ if (out.getVersion().onOrAfter(Version.V_3_2_0)) {
+ fetchProfileResult.writeTo(out);
+ }
networkTime.writeTo(out);
}
@@ -98,6 +125,10 @@ public AggregationProfileShardResult getAggregationProfileResults() {
return aggProfileShardResult;
}
+ public FetchProfileShardResult getFetchProfileResult() {
+ return fetchProfileResult;
+ }
+
public NetworkTime getNetworkTime() {
return networkTime;
}
diff --git a/server/src/main/java/org/opensearch/search/profile/Profilers.java b/server/src/main/java/org/opensearch/search/profile/Profilers.java
index a862620871f47..535a0b6d0f04d 100644
--- a/server/src/main/java/org/opensearch/search/profile/Profilers.java
+++ b/server/src/main/java/org/opensearch/search/profile/Profilers.java
@@ -37,6 +37,7 @@
import org.opensearch.search.internal.ContextIndexSearcher;
import org.opensearch.search.profile.aggregation.AggregationProfiler;
import org.opensearch.search.profile.aggregation.ConcurrentAggregationProfiler;
+import org.opensearch.search.profile.fetch.FetchProfiler;
import org.opensearch.search.profile.query.ConcurrentQueryProfileTree;
import org.opensearch.search.profile.query.ConcurrentQueryProfiler;
import org.opensearch.search.profile.query.InternalQueryProfileTree;
@@ -60,6 +61,7 @@ public final class Profilers {
private final ContextIndexSearcher searcher;
private final List queryProfilers;
private final AggregationProfiler aggProfiler;
+ private final FetchProfiler fetchProfiler;
private final boolean isConcurrentSegmentSearchEnabled;
private final Function>> customPluginMetrics;
@@ -78,6 +80,7 @@ public Profilers(
this.isConcurrentSegmentSearchEnabled = isConcurrentSegmentSearchEnabled;
this.queryProfilers = new ArrayList<>();
this.aggProfiler = isConcurrentSegmentSearchEnabled ? new ConcurrentAggregationProfiler() : new AggregationProfiler();
+ this.fetchProfiler = new FetchProfiler();
this.customPluginMetrics = customPluginMetrics;
addQueryProfiler();
}
@@ -107,4 +110,8 @@ public AggregationProfiler getAggregationProfiler() {
return aggProfiler;
}
+ public FetchProfiler getFetchProfiler() {
+ return fetchProfiler;
+ }
+
}
diff --git a/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java b/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java
index 53cf1f2739e77..1696604fe05e4 100644
--- a/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java
+++ b/server/src/main/java/org/opensearch/search/profile/SearchProfileShardResults.java
@@ -41,6 +41,8 @@
import org.opensearch.search.internal.ShardSearchRequest;
import org.opensearch.search.profile.aggregation.AggregationProfileShardResult;
import org.opensearch.search.profile.aggregation.AggregationProfiler;
+import org.opensearch.search.profile.fetch.FetchProfileShardResult;
+import org.opensearch.search.profile.fetch.FetchProfiler;
import org.opensearch.search.profile.query.QueryProfileShardResult;
import org.opensearch.search.profile.query.QueryProfiler;
@@ -117,6 +119,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endArray();
profileShardResult.getAggregationProfileResults().toXContent(builder, params);
+ profileShardResult.getFetchProfileResult().toXContent(builder, params);
builder.endObject();
}
builder.endArray().endObject();
@@ -149,6 +152,7 @@ private static void parseSearchProfileResultsEntry(XContentParser parser, Map queryProfileResults = new ArrayList<>();
AggregationProfileShardResult aggProfileShardResult = null;
+ FetchProfileShardResult fetchProfileShardResult = null;
String id = null;
String currentFieldName = null;
long inboundNetworkTime = 0;
@@ -173,6 +177,8 @@ private static void parseSearchProfileResultsEntry(XContentParser parser, Map queryProfilers = profilers.getQueryProfilers();
AggregationProfiler aggProfiler = profilers.getAggregationProfiler();
+ FetchProfiler fetchProfiler = profilers.getFetchProfiler();
List queryResults = new ArrayList<>(queryProfilers.size());
for (QueryProfiler queryProfiler : queryProfilers) {
QueryProfileShardResult result = new QueryProfileShardResult(
@@ -206,11 +219,13 @@ public static ProfileShardResult buildShardResults(Profilers profilers, ShardSea
queryResults.add(result);
}
AggregationProfileShardResult aggResults = new AggregationProfileShardResult(aggProfiler.getTree());
+ List fetchTree = fetchProfiler.getTree();
+ FetchProfileShardResult fetchResult = new FetchProfileShardResult(fetchTree);
NetworkTime networkTime = new NetworkTime(0, 0);
if (request != null) {
networkTime.setInboundNetworkTime(request.getInboundNetworkTime());
networkTime.setOutboundNetworkTime(request.getOutboundNetworkTime());
}
- return new ProfileShardResult(queryResults, aggResults, networkTime);
+ return new ProfileShardResult(queryResults, aggResults, fetchResult, networkTime);
}
}
diff --git a/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfileBreakdown.java
new file mode 100644
index 0000000000000..bb30fc6999609
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfileBreakdown.java
@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.fetch;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.search.profile.AbstractProfileBreakdown;
+import org.opensearch.search.profile.ProfileMetricUtil;
+
+/**
+ * A record of timings for the various operations that may happen during fetch execution.
+ */
+@ExperimentalApi()
+public class FetchProfileBreakdown extends AbstractProfileBreakdown {
+ public FetchProfileBreakdown() {
+ super(ProfileMetricUtil.getFetchProfileMetrics());
+ }
+}
diff --git a/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfileShardResult.java
new file mode 100644
index 0000000000000..27d32736e6726
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfileShardResult.java
@@ -0,0 +1,78 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.fetch;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.search.profile.ProfileResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;
+
+/**
+ * Represents the fetch phase profile results for a single shard.
+ */
+@ExperimentalApi()
+public class FetchProfileShardResult implements Writeable, ToXContentFragment {
+ public static final String FETCH = "fetch";
+
+ private final List fetchProfileResults;
+
+ public FetchProfileShardResult(List results) {
+ this.fetchProfileResults = Collections.unmodifiableList(results);
+ }
+
+ public FetchProfileShardResult(StreamInput in) throws IOException {
+ int profileSize = in.readVInt();
+ List tmp = new ArrayList<>(profileSize);
+ for (int j = 0; j < profileSize; j++) {
+ tmp.add(new ProfileResult(in));
+ }
+ this.fetchProfileResults = Collections.unmodifiableList(tmp);
+ }
+
+ public List getFetchProfileResults() {
+ return fetchProfileResults;
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeVInt(fetchProfileResults.size());
+ for (ProfileResult p : fetchProfileResults) {
+ p.writeTo(out);
+ }
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startArray(FETCH);
+ for (ProfileResult p : fetchProfileResults) {
+ p.toXContent(builder, params);
+ }
+ return builder.endArray();
+ }
+
+ public static FetchProfileShardResult fromXContent(XContentParser parser) throws IOException {
+ ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
+ List results = new ArrayList<>();
+ while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+ results.add(ProfileResult.fromXContent(parser));
+ }
+ return new FetchProfileShardResult(results);
+ }
+}
diff --git a/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfiler.java b/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfiler.java
new file mode 100644
index 0000000000000..34d7df9dd5649
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/fetch/FetchProfiler.java
@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.fetch;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.search.profile.ProfileResult;
+
+import java.util.List;
+
+/**
+ * Profiler for the fetch phase using a simplified profiling tree.
+ */
+@ExperimentalApi()
+public class FetchProfiler {
+ private final FlatFetchProfileTree profileTree;
+
+ /** Creates a new FetchProfiler. */
+ public FetchProfiler() {
+ this.profileTree = new FlatFetchProfileTree();
+ }
+
+ /**
+ * Get the {@link FetchProfileBreakdown} for the given element in the tree,
+ * potentially creating it if it did not exist.
+ */
+ public FetchProfileBreakdown startFetchPhase(String element) {
+ return profileTree.startFetchPhase(element);
+ }
+
+ /**
+ * Get the {@link FetchProfileBreakdown} for a fetch sub-phase under the specified parent.
+ */
+ public FetchProfileBreakdown startSubPhase(String element, String parentElement) {
+ return profileTree.startSubPhase(element, parentElement);
+ }
+
+ /**
+ * Finish profiling of the specified fetch phase.
+ */
+ public void endFetchPhase(String element) {
+ profileTree.endFetchPhase(element);
+ }
+
+ /**
+ * @return a hierarchical representation of the profiled tree.
+ */
+ public List getTree() {
+ return profileTree.getTree();
+ }
+}
diff --git a/server/src/main/java/org/opensearch/search/profile/fetch/FetchTimingType.java b/server/src/main/java/org/opensearch/search/profile/fetch/FetchTimingType.java
new file mode 100644
index 0000000000000..ce776c8c35589
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/fetch/FetchTimingType.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.fetch;
+
+import java.util.Locale;
+
+/**
+ * Timing points for fetch phase profiling.
+ */
+public enum FetchTimingType {
+ /** Time spent creating the stored fields visitor */
+ CREATE_STORED_FIELDS_VISITOR,
+ /** Time spent building fetch sub-phase processors */
+ BUILD_SUB_PHASE_PROCESSORS,
+ /** Time spent getting the specific leaf reader context for a given segment index */
+ GET_NEXT_READER,
+ /** Time spent setting next reader for each subphase processor **/
+ SET_NEXT_READER,
+ /** Time spent loading stored fields for a hit */
+ LOAD_STORED_FIELDS,
+ /** Time spent loading the document _source */
+ LOAD_SOURCE,
+ /** Time spent executing a fetch sub-phase */
+ PROCESS;
+
+ @Override
+ public String toString() {
+ return name().toLowerCase(Locale.ROOT);
+ }
+}
diff --git a/server/src/main/java/org/opensearch/search/profile/fetch/FlatFetchProfileTree.java b/server/src/main/java/org/opensearch/search/profile/fetch/FlatFetchProfileTree.java
new file mode 100644
index 0000000000000..fa5f34526c356
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/fetch/FlatFetchProfileTree.java
@@ -0,0 +1,125 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.fetch;
+
+import org.opensearch.search.profile.ProfileResult;
+import org.opensearch.search.profile.Timer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Simplified profiling tree for fetch phase operations. Each fetch phase is
+ * stored as a root with a single level of child sub phases.
+ *
+ * @opensearch.internal
+ */
+class FlatFetchProfileTree {
+ private static final Set ROOT_KEYS = Set.of(
+ FetchTimingType.CREATE_STORED_FIELDS_VISITOR.toString(),
+ FetchTimingType.CREATE_STORED_FIELDS_VISITOR.toString() + Timer.TIMING_TYPE_COUNT_SUFFIX,
+ FetchTimingType.BUILD_SUB_PHASE_PROCESSORS.toString(),
+ FetchTimingType.BUILD_SUB_PHASE_PROCESSORS.toString() + Timer.TIMING_TYPE_COUNT_SUFFIX,
+ FetchTimingType.LOAD_STORED_FIELDS.toString(),
+ FetchTimingType.LOAD_STORED_FIELDS.toString() + Timer.TIMING_TYPE_COUNT_SUFFIX,
+ FetchTimingType.LOAD_SOURCE.toString(),
+ FetchTimingType.LOAD_SOURCE.toString() + Timer.TIMING_TYPE_COUNT_SUFFIX,
+ FetchTimingType.GET_NEXT_READER.toString(),
+ FetchTimingType.GET_NEXT_READER.toString() + Timer.TIMING_TYPE_COUNT_SUFFIX
+ );
+
+ private static final Set SUB_PHASE_KEYS = Set.of(
+ FetchTimingType.PROCESS.toString(),
+ FetchTimingType.PROCESS.toString() + Timer.TIMING_TYPE_COUNT_SUFFIX,
+ FetchTimingType.SET_NEXT_READER.toString(),
+ FetchTimingType.SET_NEXT_READER.toString() + Timer.TIMING_TYPE_COUNT_SUFFIX
+ );
+
+ private static class Node {
+ final String element;
+ final FetchProfileBreakdown breakdown;
+ final List children = new ArrayList<>();
+
+ Node(String element) {
+ this.element = element;
+ this.breakdown = new FetchProfileBreakdown();
+ }
+ }
+
+ private final List roots = new ArrayList<>();
+ private final Map phaseMap = new HashMap<>();
+
+ /** Start profiling a new fetch phase and return its breakdown. */
+ FetchProfileBreakdown startFetchPhase(String element) {
+ Node node = new Node(element);
+ roots.add(node);
+ phaseMap.put(element, node);
+ return node.breakdown;
+ }
+
+ /** Start profiling a fetch sub-phase under the specified parent phase. */
+ FetchProfileBreakdown startSubPhase(String element, String parentElement) {
+ Node parent = phaseMap.get(parentElement);
+ if (parent == null) {
+ throw new IllegalStateException("Parent phase '" + parentElement + "' does not exist for sub-phase '" + element + "'");
+ }
+ Node child = new Node(element);
+ parent.children.add(child);
+ return child.breakdown;
+ }
+
+ /**
+ * Finish profiling of the specified fetch phase.
+ */
+ void endFetchPhase(String element) {
+ phaseMap.remove(element);
+ }
+
+ /**
+ * Build the profile results tree for serialization.
+ */
+ List getTree() {
+ List results = new ArrayList<>(roots.size());
+ for (Node root : roots) {
+ results.add(toProfileResult(root, true));
+ }
+ return results;
+ }
+
+ private ProfileResult toProfileResult(Node node, boolean isRoot) {
+ List children = new ArrayList<>(node.children.size());
+ for (Node child : node.children) {
+ children.add(toProfileResult(child, false));
+ }
+ Map raw = node.breakdown.toBreakdownMap();
+ Map filtered = filterBreakdown(raw, isRoot);
+ return new ProfileResult(node.element, node.element, filtered, node.breakdown.toDebugMap(), inclusiveTime(node), children);
+ }
+
+ private long inclusiveTime(Node node) {
+ long total = node.breakdown.toNodeTime();
+ for (Node child : node.children) {
+ total += inclusiveTime(child);
+ }
+ return total;
+ }
+
+ private Map filterBreakdown(Map raw, boolean isRoot) {
+ Set allowed = isRoot ? ROOT_KEYS : SUB_PHASE_KEYS;
+ Map map = new TreeMap<>();
+ for (String key : allowed) {
+ map.put(key, raw.getOrDefault(key, 0L));
+ }
+ return map;
+ }
+}
diff --git a/server/src/main/java/org/opensearch/search/profile/fetch/package-info.java b/server/src/main/java/org/opensearch/search/profile/fetch/package-info.java
new file mode 100644
index 0000000000000..ab5f1179e424e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/fetch/package-info.java
@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/**
+ * Profiling classes for the fetch portion of a search request.
+ */
+package org.opensearch.search.profile.fetch;
diff --git a/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java b/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java
index 4f03dfd47c512..56fd63f00357b 100644
--- a/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java
@@ -32,11 +32,20 @@
package org.opensearch.search.fetch;
+import org.opensearch.action.OriginalIndices;
+import org.opensearch.action.search.SearchShardTask;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.core.tasks.TaskCancelledException;
import org.opensearch.index.fieldvisitor.CustomFieldsVisitor;
import org.opensearch.index.fieldvisitor.FieldsVisitor;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.search.SearchShardTarget;
import org.opensearch.search.fetch.subphase.FetchSourceContext;
import org.opensearch.search.internal.SearchContext;
+import org.opensearch.search.lookup.SearchLookup;
import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.TestSearchContext;
import java.util.ArrayList;
import java.util.Collections;
@@ -45,6 +54,7 @@
import java.util.Map;
import java.util.Set;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -116,4 +126,48 @@ public void testFieldsVisitorsInFetchPhase() {
}
+ public void testTaskCancellationDuringFetch() {
+ FetchPhase fetchPhase = new FetchPhase(Collections.emptyList());
+
+ SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+ SearchContext context = mock(SearchContext.class);
+ when(context.isCancelled()).thenReturn(false, true);
+ when(context.getTask()).thenReturn(task);
+
+ when(context.docIdsToLoadSize()).thenReturn(1);
+ when(context.docIdsToLoad()).thenReturn(new int[] { 0 });
+ when(context.docIdsToLoadFrom()).thenReturn(0);
+ when(context.hasScriptFields()).thenReturn(false);
+ when(context.hasFetchSourceContext()).thenReturn(false);
+ when(context.storedFieldsContext()).thenReturn(null);
+ when(context.fetchSourceContext(any(FetchSourceContext.class))).thenReturn(null);
+ when(context.fetchSourceContext()).thenReturn(new FetchSourceContext(true));
+
+ QueryShardContext queryShardContext = mock(QueryShardContext.class);
+ SearchLookup lookup = new SearchLookup(mock(MapperService.class), (ft, sl) -> null);
+ when(queryShardContext.newFetchLookup()).thenReturn(lookup);
+ when(context.getQueryShardContext()).thenReturn(queryShardContext);
+
+ SearchShardTarget target = new SearchShardTarget("node", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE);
+ when(context.shardTarget()).thenReturn(target);
+
+ task.cancel("test");
+
+ TaskCancelledException ex = expectThrows(TaskCancelledException.class, () -> fetchPhase.execute(context));
+ assertEquals("cancelled task with reason: test", ex.getMessage());
+ }
+
+ public void testExecuteCancelledTaskThrows() {
+ FetchPhase fetchPhase = new FetchPhase(new ArrayList<>());
+ TestSearchContext context = new TestSearchContext(null, null, null);
+
+ SearchShardTask task = mock(SearchShardTask.class);
+ when(task.isCancelled()).thenReturn(true);
+ when(task.getReasonCancelled()).thenReturn("test reason");
+ context.setTask(task);
+
+ TaskCancelledException ex = expectThrows(TaskCancelledException.class, () -> fetchPhase.execute(context));
+ assertEquals("cancelled task with reason: test reason", ex.getMessage());
+ }
+
}
diff --git a/server/src/test/java/org/opensearch/search/fetch/FetchProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/fetch/FetchProfilePhaseTests.java
new file mode 100644
index 0000000000000..cf4161e9d0562
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/fetch/FetchProfilePhaseTests.java
@@ -0,0 +1,848 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.fetch;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
+import org.opensearch.core.common.text.Text;
+import org.opensearch.index.mapper.DocumentMapper;
+import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.query.ParsedQuery;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.index.shard.IndexShardTestCase;
+import org.opensearch.search.DocValueFormat;
+import org.opensearch.search.SearchHit;
+import org.opensearch.search.fetch.subphase.ExplainPhase;
+import org.opensearch.search.fetch.subphase.FetchDocValuesContext;
+import org.opensearch.search.fetch.subphase.FetchDocValuesPhase;
+import org.opensearch.search.fetch.subphase.FetchFieldsContext;
+import org.opensearch.search.fetch.subphase.FetchFieldsPhase;
+import org.opensearch.search.fetch.subphase.FetchScorePhase;
+import org.opensearch.search.fetch.subphase.FetchSourceContext;
+import org.opensearch.search.fetch.subphase.FetchSourcePhase;
+import org.opensearch.search.fetch.subphase.FetchVersionPhase;
+import org.opensearch.search.fetch.subphase.FieldAndFormat;
+import org.opensearch.search.fetch.subphase.InnerHitsContext;
+import org.opensearch.search.fetch.subphase.MatchedQueriesPhase;
+import org.opensearch.search.fetch.subphase.ScriptFieldsContext;
+import org.opensearch.search.fetch.subphase.SeqNoPrimaryTermPhase;
+import org.opensearch.search.fetch.subphase.highlight.FieldHighlightContext;
+import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.opensearch.search.fetch.subphase.highlight.HighlightField;
+import org.opensearch.search.fetch.subphase.highlight.HighlightPhase;
+import org.opensearch.search.fetch.subphase.highlight.Highlighter;
+import org.opensearch.search.fetch.subphase.highlight.SearchHighlightContext;
+import org.opensearch.search.internal.ContextIndexSearcher;
+import org.opensearch.search.internal.SearchContext;
+import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.lookup.SourceLookup;
+import org.opensearch.search.profile.ProfileResult;
+import org.opensearch.search.profile.Profilers;
+import org.opensearch.search.profile.fetch.FetchProfiler;
+import org.opensearch.search.profile.fetch.FetchTimingType;
+import org.opensearch.search.query.QuerySearchResult;
+import org.opensearch.search.sort.SortAndFormats;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class FetchProfilePhaseTests extends IndexShardTestCase {
+ private IndexShard indexShard;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ indexShard = newShard(true);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ closeShards(indexShard);
+ }
+
+ private static class TestDocumentBuilder {
+ private final List documents = new ArrayList<>();
+
+ TestDocumentBuilder addDocuments(int count, boolean includeSource) {
+ for (int i = 0; i < count; i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", String.valueOf(i), Field.Store.YES));
+ doc.add(new TextField("content", "test content " + i, Field.Store.YES));
+
+ if (includeSource) {
+ String sourceJson = "{\"id\":" + i + ",\"content\":\"test content " + i + "\"}";
+ doc.add(new org.apache.lucene.document.StoredField("_source", sourceJson.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ documents.add(doc);
+ }
+ return this;
+ }
+
+ List build() {
+ return documents;
+ }
+ }
+
+ private static class TimingAssertions {
+ private final Map breakdown;
+
+ TimingAssertions(Map breakdown) {
+ this.breakdown = breakdown;
+ }
+
+ TimingAssertions assertTimingPresent(FetchTimingType timingType) {
+ String timingKey = timingType.toString();
+ String countKey = timingKey + "_count";
+ String startTimeKey = timingKey + "_start_time";
+
+ assertTrue("Timing key should be present: " + timingKey, breakdown.containsKey(timingKey));
+ assertTrue("Count key should be present: " + countKey, breakdown.containsKey(countKey));
+
+ assertThat("Timing should be geq 1: " + timingKey, breakdown.get(timingKey), greaterThanOrEqualTo(1L));
+ assertThat("Count should be non-negative: " + countKey, breakdown.get(countKey), greaterThanOrEqualTo(0L));
+ return this;
+ }
+
+ TimingAssertions assertBreakdownNotEmpty() {
+ assertThat("Profile breakdown should not be empty", breakdown.size(), greaterThan(0));
+ return this;
+ }
+ }
+
+ private static class SearchContextBuilder {
+ private final IndexReader reader;
+ private final int[] docIds;
+ private final IndexShard indexShard;
+
+ private boolean enableSourceLoading = false;
+ private boolean enableStoredFields = false;
+ private List storedFieldNames = Collections.emptyList();
+ private boolean enableScriptFields = false;
+ private ScriptFieldsContext scriptFieldsContext = null;
+
+ private boolean enableExplain = false;
+ private boolean enableVersion = false;
+ private boolean enableSeqNoPrimaryTerm = false;
+ private boolean enableFetchScore = false;
+
+ private boolean enableDocValues = false;
+ private List docValueFields = Collections.emptyList();
+
+ private boolean enableFetchFields = false;
+ private List fetchFields = Collections.emptyList();
+
+ private ParsedQuery parsedQuery = null;
+ private Query query = new MatchAllDocsQuery();
+
+ private SearchHighlightContext highlightContext = null;
+ private QueryShardContext externalQueryShardContext = null;
+
+ SearchContextBuilder(IndexReader reader, int[] docIds, IndexShard indexShard) {
+ this.reader = reader;
+ this.docIds = docIds;
+ this.indexShard = indexShard;
+ }
+
+ SearchContextBuilder withSourceLoading() {
+ this.enableSourceLoading = true;
+ return this;
+ }
+
+ SearchContextBuilder withStoredFields(String... fieldNames) {
+ this.enableStoredFields = true;
+ this.storedFieldNames = Arrays.asList(fieldNames);
+ return this;
+ }
+
+ SearchContextBuilder withScriptFields(ScriptFieldsContext ctx) {
+ this.enableScriptFields = true;
+ this.scriptFieldsContext = ctx;
+ return this;
+ }
+
+ SearchContextBuilder withExplain() {
+ this.enableExplain = true;
+ return this;
+ }
+
+ SearchContextBuilder withVersion() {
+ this.enableVersion = true;
+ return this;
+ }
+
+ SearchContextBuilder withSeqNoPrimaryTerm() {
+ this.enableSeqNoPrimaryTerm = true;
+ return this;
+ }
+
+ SearchContextBuilder withFetchScore() {
+ this.enableFetchScore = true;
+ return this;
+ }
+
+ SearchContextBuilder withDocValues(FieldAndFormat... fields) {
+ this.enableDocValues = true;
+ this.docValueFields = Arrays.asList(fields);
+ return this;
+ }
+
+ SearchContextBuilder withFetchFields(FieldAndFormat... fields) {
+ this.enableFetchFields = true;
+ this.fetchFields = Arrays.asList(fields);
+ return this;
+ }
+
+ SearchContextBuilder withParsedQuery(ParsedQuery pq) {
+ this.parsedQuery = pq;
+ return this;
+ }
+
+ SearchContextBuilder withQuery(Query query) {
+ this.query = query;
+ return this;
+ }
+
+ SearchContextBuilder withHighlight(SearchHighlightContext hc) {
+ this.highlightContext = hc;
+ return this;
+ }
+
+ SearchContextBuilder withQueryShardContext(QueryShardContext qsc) {
+ this.externalQueryShardContext = qsc;
+ return this;
+ }
+
+ SearchContext build() throws IOException {
+ SearchContext context = mock(SearchContext.class);
+
+ // Basic document loading configuration
+ when(context.docIdsToLoadSize()).thenReturn(docIds.length);
+ when(context.docIdsToLoad()).thenReturn(docIds);
+ when(context.docIdsToLoadFrom()).thenReturn(0);
+ when(context.isCancelled()).thenReturn(false);
+
+ // Script fields configuration
+ when(context.hasScriptFields()).thenReturn(enableScriptFields);
+
+ if (enableScriptFields) {
+ when(context.scriptFields()).thenReturn(scriptFieldsContext);
+ }
+
+ // Basic query configuration
+ when(context.query()).thenReturn(query);
+ when(context.parsedQuery()).thenReturn(parsedQuery);
+ when(context.explain()).thenReturn(enableExplain);
+ when(context.version()).thenReturn(enableVersion);
+ when(context.seqNoAndPrimaryTerm()).thenReturn(enableSeqNoPrimaryTerm);
+
+ // Source loading configuration
+ when(context.hasFetchSourceContext()).thenReturn(enableSourceLoading);
+ when(context.sourceRequested()).thenReturn(enableSourceLoading);
+ if (enableSourceLoading) {
+ when(context.fetchSourceContext()).thenReturn(new FetchSourceContext(true));
+ }
+
+ // Stored fields configuration
+ if (enableStoredFields && !storedFieldNames.isEmpty()) {
+ StoredFieldsContext storedFieldsContext = StoredFieldsContext.fromList(storedFieldNames);
+ when(context.storedFieldsContext()).thenReturn(storedFieldsContext);
+ } else {
+ when(context.storedFieldsContext()).thenReturn(null);
+ }
+
+ if (enableDocValues) {
+ when(context.docValuesContext()).thenReturn(new FetchDocValuesContext(docValueFields));
+ } else {
+ when(context.docValuesContext()).thenReturn(null);
+ }
+
+ if (enableFetchFields) {
+ when(context.fetchFieldsContext()).thenReturn(new FetchFieldsContext(fetchFields));
+ } else {
+ when(context.fetchFieldsContext()).thenReturn(null);
+ }
+
+ if (highlightContext != null) {
+ when(context.highlight()).thenReturn(highlightContext);
+ } else {
+ when(context.highlight()).thenReturn(null);
+ }
+
+ if (enableFetchScore) {
+ Sort sort = new Sort(SortField.FIELD_SCORE);
+ SortAndFormats sf = new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW });
+ when(context.sort()).thenReturn(sf);
+ when(context.trackScores()).thenReturn(true);
+ } else {
+ when(context.sort()).thenReturn(null);
+ when(context.trackScores()).thenReturn(false);
+ }
+
+ // Query result setup
+ QuerySearchResult queryResult = new QuerySearchResult();
+ ScoreDoc[] scoreDocs = new ScoreDoc[docIds.length];
+ for (int i = 0; i < docIds.length; i++) {
+ scoreDocs[i] = new ScoreDoc(docIds[i], 1.0f);
+ }
+ TopDocsAndMaxScore topDocsAndMaxScore = new TopDocsAndMaxScore(
+ new org.apache.lucene.search.TopDocs(new TotalHits(docIds.length, TotalHits.Relation.EQUAL_TO), scoreDocs),
+ 1.0f
+ );
+ queryResult.topDocs(topDocsAndMaxScore, null);
+ when(context.queryResult()).thenReturn(queryResult);
+
+ // Search lookup setup
+ QueryShardContext queryShardContext = externalQueryShardContext != null
+ ? externalQueryShardContext
+ : mock(QueryShardContext.class);
+ SearchLookup searchLookup = mock(SearchLookup.class);
+ SourceLookup sourceLookup = new SourceLookup();
+ when(searchLookup.source()).thenReturn(sourceLookup);
+ when(queryShardContext.newFetchLookup()).thenReturn(searchLookup);
+ when(context.getQueryShardContext()).thenReturn(queryShardContext);
+
+ // Mapper service setup
+ MapperService mapperService = mock(MapperService.class);
+ when(mapperService.hasNested()).thenReturn(false);
+ DocumentMapper documentMapper = mock(DocumentMapper.class);
+ when(documentMapper.typeText()).thenReturn(new Text("_doc"));
+
+ // Mock the sourceMapper
+ org.opensearch.index.mapper.SourceFieldMapper sourceFieldMapper = mock(org.opensearch.index.mapper.SourceFieldMapper.class);
+ when(sourceFieldMapper.enabled()).thenReturn(true);
+ when(documentMapper.sourceMapper()).thenReturn(sourceFieldMapper);
+
+ when(mapperService.documentMapper()).thenReturn(documentMapper);
+ when(context.mapperService()).thenReturn(mapperService);
+
+ // Index searcher setup
+ ContextIndexSearcher searcher = new ContextIndexSearcher(
+ reader,
+ IndexSearcher.getDefaultSimilarity(),
+ IndexSearcher.getDefaultQueryCache(),
+ IndexSearcher.getDefaultQueryCachingPolicy(),
+ true,
+ null,
+ context
+ );
+ when(context.searcher()).thenReturn(searcher);
+ when(context.indexShard()).thenReturn(indexShard);
+
+ // Fetch result setup
+ FetchSearchResult fetchResult = new FetchSearchResult();
+ when(context.fetchResult()).thenReturn(fetchResult);
+
+ // Profiling setup
+ Profilers profilers = new Profilers(searcher, false);
+ when(context.getProfilers()).thenReturn(profilers);
+
+ return context;
+ }
+ }
+
+ // Helper method to execute fetch phase and get profile results
+ private ProfileResult executeFetchPhaseAndGetProfile(SearchContext context, List subPhases) throws IOException {
+ FetchPhase fetchPhase = new FetchPhase(subPhases);
+ fetchPhase.execute(context);
+
+ FetchProfiler fetchProfiler = context.getProfilers().getFetchProfiler();
+ List profileResults = fetchProfiler.getTree();
+ assertThat(profileResults, hasSize(greaterThanOrEqualTo(1)));
+
+ return profileResults.get(0);
+ }
+
+ // Helper method to index documents and get their IDs
+ private int[] indexDocumentsAndGetIds(Directory dir, List docs, int maxResults) throws IOException {
+ try (RandomIndexWriter w = new RandomIndexWriter(random(), dir, new IndexWriterConfig(new StandardAnalyzer()))) {
+ for (Document doc : docs) {
+ w.addDocument(doc);
+ }
+ }
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), maxResults);
+
+ int[] docIds = new int[Math.min(maxResults, topDocs.scoreDocs.length)];
+ for (int i = 0; i < docIds.length; i++) {
+ docIds[i] = topDocs.scoreDocs[i].doc;
+ }
+ return docIds;
+ }
+ }
+
+ // Simple highlighter
+ private static class StubHighlighter implements Highlighter {
+ @Override
+ public HighlightField highlight(FieldHighlightContext fieldContext) {
+ return null;
+ }
+
+ @Override
+ public boolean canHighlight(MappedFieldType fieldType) {
+ return true;
+ }
+ }
+
+ // Minimal inner hit sub context
+ private static class DummyInnerHitSubContext extends InnerHitsContext.InnerHitSubContext {
+ DummyInnerHitSubContext(String name, SearchContext context) {
+ super(name, context);
+ }
+
+ @Override
+ public TopDocsAndMaxScore topDocs(SearchHit hit) {
+ ScoreDoc[] docs = new ScoreDoc[] { new ScoreDoc(hit.docId(), 1.0f) };
+ TopDocs topDocs = new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), docs);
+ return new TopDocsAndMaxScore(topDocs, 1.0f);
+ }
+ }
+
+ public void testRootTiming() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(3, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 2);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .build();
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, Collections.emptyList());
+
+ new TimingAssertions(profile.getTimeBreakdown()).assertBreakdownNotEmpty()
+ .assertTimingPresent(FetchTimingType.CREATE_STORED_FIELDS_VISITOR)
+ .assertTimingPresent(FetchTimingType.LOAD_SOURCE)
+ .assertTimingPresent(FetchTimingType.LOAD_STORED_FIELDS)
+ .assertTimingPresent(FetchTimingType.BUILD_SUB_PHASE_PROCESSORS)
+ .assertTimingPresent(FetchTimingType.GET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testExplainPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery(), Map.of("named", new MatchAllDocsQuery()));
+
+ // Create a search context with only explain enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withExplain() // Only enable explain
+ .withParsedQuery(pq)
+ .build();
+
+ // Only include the ExplainPhase in the sub-phases list
+ List subPhases = Collections.singletonList(new ExplainPhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that ExplainPhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for ExplainPhase", children.containsKey("ExplainPhase"));
+ new TimingAssertions(children.get("ExplainPhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testFetchSourcePhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ // Create a search context with only source loading enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading() // Enable source loading
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .build();
+
+ // Only include the FetchSourcePhase in the sub-phases list
+ List subPhases = Collections.singletonList(new FetchSourcePhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that FetchSourcePhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for FetchSourcePhase", children.containsKey("FetchSourcePhase"));
+ new TimingAssertions(children.get("FetchSourcePhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testFetchDocValuesPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ // Create a search context with doc values enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .withDocValues(new FieldAndFormat("id", null)) // Enable doc values for the id field
+ .build();
+
+ // Only include the FetchDocValuesPhase in the sub-phases list
+ List subPhases = Collections.singletonList(new FetchDocValuesPhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that FetchDocValuesPhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for FetchDocValuesPhase", children.containsKey("FetchDocValuesPhase"));
+ new TimingAssertions(children.get("FetchDocValuesPhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testFetchFieldsPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ // Create a search context with fetch fields enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .withFetchFields(new FieldAndFormat("content", null)) // Enable fetch fields for the content field
+ .build();
+
+ // Only include the FetchFieldsPhase in the sub-phases list
+ List subPhases = Collections.singletonList(new FetchFieldsPhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that FetchFieldsPhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for FetchFieldsPhase", children.containsKey("FetchFieldsPhase"));
+ new TimingAssertions(children.get("FetchFieldsPhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testFetchVersionPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ // Create a search context with version enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .withVersion() // Enable version
+ .build();
+
+ // Only include the FetchVersionPhase in the sub-phases list
+ List subPhases = Collections.singletonList(new FetchVersionPhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that FetchVersionPhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for FetchVersionPhase", children.containsKey("FetchVersionPhase"));
+ new TimingAssertions(children.get("FetchVersionPhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testSeqNoPrimaryTermPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ // Create a search context with sequence number and primary term enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .withSeqNoPrimaryTerm() // Enable sequence number and primary term
+ .build();
+
+ // Only include the SeqNoPrimaryTermPhase in the sub-phases list
+ List subPhases = Collections.singletonList(new SeqNoPrimaryTermPhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that SeqNoPrimaryTermPhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for SeqNoPrimaryTermPhase", children.containsKey("SeqNoPrimaryTermPhase"));
+ new TimingAssertions(children.get("SeqNoPrimaryTermPhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testMatchedQueriesPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+
+ // Create a parsed query with named queries for the MatchedQueriesPhase to extract
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery(), Map.of("test_query", new MatchAllDocsQuery()));
+
+ // Create a search context with the parsed query containing named queries
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .build();
+
+ // Only include the MatchedQueriesPhase in the sub-phases list
+ List subPhases = Collections.singletonList(new MatchedQueriesPhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that MatchedQueriesPhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for MatchedQueriesPhase", children.containsKey("MatchedQueriesPhase"));
+ new TimingAssertions(children.get("MatchedQueriesPhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testHighlightPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ // Create highlight builder with stub highlighter
+ HighlightBuilder hb = new HighlightBuilder();
+ hb.field(new HighlightBuilder.Field("content").highlighterType("stub"));
+ SearchHighlightContext highlight = hb.build(qsc);
+
+ // Create a search context with highlighting enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .withHighlight(highlight)
+ .build();
+
+ // Only include the HighlightPhase in the sub-phases list with the stub highlighter
+ List subPhases = Collections.singletonList(
+ new HighlightPhase(Collections.singletonMap("stub", new StubHighlighter()))
+ );
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that HighlightPhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for HighlightPhase", children.containsKey("HighlightPhase"));
+ new TimingAssertions(children.get("HighlightPhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testFetchScorePhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ // Create a search context with score tracking enabled
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .withFetchScore() // Enable score fetching
+ .build();
+
+ // Only include the FetchScorePhase in the sub-phases list
+ List subPhases = Collections.singletonList(new FetchScorePhase());
+
+ ProfileResult profile = executeFetchPhaseAndGetProfile(context, subPhases);
+
+ // Verify the profile results
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ // Check that FetchScorePhase is present and has timing information
+ assertEquals(1, children.size());
+ assertTrue("Missing profile for FetchScorePhase", children.containsKey("FetchScorePhase"));
+ new TimingAssertions(children.get("FetchScorePhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+
+ public void testInnerHitsPhaseProfiling() throws Exception {
+ try (Directory dir = newDirectory()) {
+ List docs = new TestDocumentBuilder().addDocuments(1, true).build();
+ int[] docIds = indexDocumentsAndGetIds(dir, docs, 1);
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ QueryShardContext qsc = mock(QueryShardContext.class);
+ ParsedQuery pq = new ParsedQuery(new MatchAllDocsQuery());
+
+ SearchContext context = new SearchContextBuilder(reader, docIds, indexShard).withSourceLoading()
+ .withStoredFields("_source")
+ .withQueryShardContext(qsc)
+ .withParsedQuery(pq)
+ .build();
+
+ InnerHitsContext.InnerHitSubContext innerContext = new DummyInnerHitSubContext("inner", context);
+ InnerHitsContext innerHits = new InnerHitsContext();
+ innerHits.addInnerHitDefinition(innerContext);
+ when(context.innerHits()).thenReturn(innerHits);
+
+ List subPhases = Collections.singletonList(new FetchSourcePhase());
+
+ FetchPhase fetchPhase = new FetchPhase(subPhases);
+ fetchPhase.execute(context);
+
+ FetchProfiler fetchProfiler = context.getProfilers().getFetchProfiler();
+ List profileResults = fetchProfiler.getTree();
+ assertThat(profileResults, hasSize(1));
+
+ ProfileResult profile = profileResults.get(0);
+
+ Map children = new HashMap<>();
+ for (ProfileResult child : profile.getProfiledChildren()) {
+ children.put(child.getQueryName(), child);
+ }
+
+ assertEquals(1, children.size());
+ assertFalse(children.containsKey("InnerHitsPhase"));
+ assertTrue(children.containsKey("FetchSourcePhase"));
+
+ new TimingAssertions(children.get("FetchSourcePhase").getTimeBreakdown()).assertTimingPresent(FetchTimingType.PROCESS)
+ .assertTimingPresent(FetchTimingType.SET_NEXT_READER);
+ }
+ }
+ }
+}
diff --git a/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java b/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java
index 71511652c0856..e26ba35ff1523 100644
--- a/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java
+++ b/server/src/test/java/org/opensearch/search/profile/SearchProfileShardResultsTests.java
@@ -32,12 +32,16 @@
package org.opensearch.search.profile;
+import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.search.profile.aggregation.AggregationProfileShardResult;
import org.opensearch.search.profile.aggregation.AggregationProfileShardResultTests;
+import org.opensearch.search.profile.fetch.FetchProfileShardResult;
import org.opensearch.search.profile.query.QueryProfileShardResult;
import org.opensearch.search.profile.query.QueryProfileShardResultTests;
import org.opensearch.test.OpenSearchTestCase;
@@ -69,10 +73,16 @@ public static SearchProfileShardResults createTestItem() {
queryProfileResults.add(QueryProfileShardResultTests.createTestItem());
}
AggregationProfileShardResult aggProfileShardResult = AggregationProfileShardResultTests.createTestItem(1);
+ List fetchResults = new ArrayList<>();
+ int fetchItems = rarely() ? 0 : randomIntBetween(1, 2);
+ for (int f = 0; f < fetchItems; f++) {
+ fetchResults.add(ProfileResultTests.createTestItem(1, false));
+ }
+ FetchProfileShardResult fetchProfileShardResult = new FetchProfileShardResult(fetchResults);
NetworkTime networkTime = new NetworkTime(inboundTime, outboundTime);
searchProfileResults.put(
randomAlphaOfLengthBetween(5, 10),
- new ProfileShardResult(queryProfileResults, aggProfileShardResult, networkTime)
+ new ProfileShardResult(queryProfileResults, aggProfileShardResult, fetchProfileShardResult, networkTime)
);
}
return new SearchProfileShardResults(searchProfileResults);
@@ -120,4 +130,40 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws
}
+ public void testFromXContentWithoutFetchSection() throws IOException {
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject();
+ builder.startObject(SearchProfileShardResults.PROFILE_FIELD);
+ builder.startArray("shards");
+ builder.startObject();
+ builder.field("id", "shard1");
+ builder.field(SearchProfileShardResults.INBOUND_NETWORK_FIELD, 0);
+ builder.field(SearchProfileShardResults.OUTBOUND_NETWORK_FIELD, 0);
+ builder.startArray("searches");
+ builder.endArray();
+ builder.startArray(AggregationProfileShardResult.AGGREGATIONS);
+ builder.endArray();
+ // no fetch section
+ builder.endObject();
+ builder.endArray();
+ builder.endObject();
+ builder.endObject();
+
+ SearchProfileShardResults parsed;
+ try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+ ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+ ensureFieldName(parser, parser.nextToken(), SearchProfileShardResults.PROFILE_FIELD);
+ ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+ parsed = SearchProfileShardResults.fromXContent(parser);
+ assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+ assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+ assertNull(parser.nextToken());
+ }
+
+ assertEquals(1, parsed.getShardResults().size());
+ ProfileShardResult shardResult = parsed.getShardResults().get("shard1");
+ assertNotNull(shardResult);
+ assertTrue(shardResult.getFetchProfileResult().getFetchProfileResults().isEmpty());
+ }
+
}
diff --git a/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfileBreakdownTests.java b/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfileBreakdownTests.java
new file mode 100644
index 0000000000000..04239ab7f667b
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfileBreakdownTests.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.fetch;
+
+import org.opensearch.search.profile.Timer;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.Map;
+
+import static org.opensearch.search.profile.Timer.TIMING_TYPE_COUNT_SUFFIX;
+import static org.opensearch.search.profile.Timer.TIMING_TYPE_START_TIME_SUFFIX;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class FetchProfileBreakdownTests extends OpenSearchTestCase {
+
+ public void testBreakdownMap() {
+ FetchProfileBreakdown breakdown = new FetchProfileBreakdown();
+ for (FetchTimingType type : FetchTimingType.values()) {
+ Timer t = breakdown.getTimer(type);
+ t.start();
+ t.stop();
+ }
+ Map map = breakdown.toBreakdownMap();
+ assertEquals(FetchTimingType.values().length * 3, map.size());
+ for (FetchTimingType type : FetchTimingType.values()) {
+ String key = type.toString();
+ assertThat(map.get(key), greaterThan(0L));
+ assertThat(map.get(key + TIMING_TYPE_COUNT_SUFFIX), equalTo(1L));
+ assertThat(map.get(key + TIMING_TYPE_START_TIME_SUFFIX), greaterThan(0L));
+ }
+ long total = breakdown.toNodeTime();
+ long sum = 0;
+ for (FetchTimingType type : FetchTimingType.values()) {
+ sum += map.get(type.toString());
+ }
+ assertEquals(sum, total);
+ }
+}
diff --git a/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfileShardResultTests.java b/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfileShardResultTests.java
new file mode 100644
index 0000000000000..d5beeacf202fb
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfileShardResultTests.java
@@ -0,0 +1,64 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.search.profile.fetch;
+
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.core.xcontent.XContentParserUtils;
+import org.opensearch.search.profile.ProfileResult;
+import org.opensearch.search.profile.ProfileResultTests;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.opensearch.core.xcontent.XContentHelper.toXContent;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
+
+public class FetchProfileShardResultTests extends OpenSearchTestCase {
+
+ public static FetchProfileShardResult createTestItem() {
+ int size = randomIntBetween(0, 5);
+ List results = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ results.add(ProfileResultTests.createTestItem(1, false));
+ }
+ return new FetchProfileShardResult(results);
+ }
+
+ public void testFromXContent() throws IOException {
+ FetchProfileShardResult profileResult = createTestItem();
+ XContentType xContentType = randomFrom(XContentType.values());
+ boolean humanReadable = randomBoolean();
+ BytesReference originalBytes = toShuffledXContent(profileResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+ FetchProfileShardResult parsed;
+ try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+ XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+ XContentParserUtils.ensureFieldName(parser, parser.nextToken(), FetchProfileShardResult.FETCH);
+ XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser);
+ parsed = FetchProfileShardResult.fromXContent(parser);
+ assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+ assertNull(parser.nextToken());
+ }
+ assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+ }
+
+ public void testGetFetchProfileResults() {
+ List expected = new ArrayList<>();
+ expected.add(new ProfileResult("type1", "desc1", Map.of("key1", 1L), Map.of(), 10L, List.of()));
+ expected.add(new ProfileResult("type2", "desc2", Map.of("key2", 2L), Map.of(), 20L, List.of()));
+
+ FetchProfileShardResult shardResult = new FetchProfileShardResult(expected);
+
+ assertEquals(expected, shardResult.getFetchProfileResults());
+ }
+}
diff --git a/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfilerTests.java
new file mode 100644
index 0000000000000..8caea1730b542
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/profile/fetch/FetchProfilerTests.java
@@ -0,0 +1,99 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+package org.opensearch.search.profile.fetch;
+
+import org.opensearch.search.profile.ProfileResult;
+import org.opensearch.search.profile.Timer;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.opensearch.search.profile.Timer.TIMING_TYPE_COUNT_SUFFIX;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class FetchProfilerTests extends OpenSearchTestCase {
+
+ public void testBasicProfiling() {
+ FetchProfiler profiler = new FetchProfiler();
+ FetchProfileBreakdown root = profiler.startFetchPhase("fetch");
+ for (FetchTimingType type : FetchTimingType.values()) {
+ if (type == FetchTimingType.PROCESS || type == FetchTimingType.SET_NEXT_READER) {
+ continue;
+ }
+ Timer t = root.getTimer(type);
+ t.start();
+ t.stop();
+ }
+
+ FetchProfileBreakdown child = profiler.startSubPhase("phase", "fetch");
+ Timer ct = child.getTimer(FetchTimingType.PROCESS);
+ ct.start();
+ ct.stop();
+ profiler.endFetchPhase("fetch");
+
+ List results = profiler.getTree();
+ assertEquals(1, results.size());
+ ProfileResult profileResult = results.get(0);
+ assertEquals("fetch", profileResult.getQueryName());
+ Map map = profileResult.getTimeBreakdown();
+ assertEquals(10, map.size());
+ long sum = 0;
+ for (FetchTimingType type : FetchTimingType.values()) {
+ if (type == FetchTimingType.PROCESS || type == FetchTimingType.SET_NEXT_READER) {
+ continue;
+ }
+ String key = type.toString();
+ assertThat(map.get(key), greaterThan(0L));
+ assertThat(map.get(key + TIMING_TYPE_COUNT_SUFFIX), equalTo(1L));
+ sum += map.get(key);
+ }
+ long childTime = ct.getApproximateTiming();
+ assertEquals(sum + childTime, profileResult.getTime());
+ assertFalse(map.containsKey(FetchTimingType.PROCESS.toString()));
+ assertFalse(map.containsKey(FetchTimingType.SET_NEXT_READER.toString()));
+ }
+
+ public void testTimerAggregation() {
+ FetchProfileBreakdown breakdown = new FetchProfileBreakdown();
+ Timer timer = breakdown.getTimer(FetchTimingType.PROCESS);
+ timer.start();
+ timer.stop();
+ timer.start();
+ timer.stop();
+ Map map = breakdown.toBreakdownMap();
+ assertThat(map.get(FetchTimingType.PROCESS.toString()), greaterThan(0L));
+ assertThat(map.get(FetchTimingType.PROCESS + TIMING_TYPE_COUNT_SUFFIX), equalTo(2L));
+ assertThat(breakdown.toNodeTime(), equalTo(map.get(FetchTimingType.PROCESS.toString())));
+ }
+}