Skip to content

Commit 26fd3d9

Browse files
sandeshkr419sandeshkr419
committed
startree node,indices,shards stats
Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>
1 parent efc3e5d commit 26fd3d9

File tree

9 files changed

+162
-12
lines changed

9 files changed

+162
-12
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2020
- Add functionality for plugins to inject QueryCollectorContext during QueryPhase ([#18637](https://github.com/opensearch-project/OpenSearch/pull/18637))
2121
- Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460))
2222
- Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530))
23-
- Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
23+
- Add Semantic Version field type mapper and extensive unit tests ([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
24+
- [Star-Tree] Add star-tree search related stats ([#18707](https://github.com/opensearch-project/OpenSearch/pull/18707))
2425

2526
### Changed
2627
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"Help":
22
- skip:
33
version: " - 2.13.99"
4-
reason: search idle reactivate count total is only added in 3.0.0
4+
reason: star-tree search stats is only added in 3.1.0
55
features: node_selector
66
- do:
77
cat.shards:
@@ -70,6 +70,8 @@
7070
search.concurrent_query_time .+ \n
7171
search.concurrent_query_total .+ \n
7272
search.concurrent_avg_slice_count .+ \n
73+
search.startree_query_total .+ \n
74+
search.startree_query_time_in_millis .+ \n
7375
search.scroll_current .+ \n
7476
search.scroll_time .+ \n
7577
search.scroll_total .+ \n

‎server/src/main/java/org/opensearch/index/search/stats/SearchStats.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,9 @@ public static class Stats implements Writeable, ToXContentFragment {
166166

167167
private long searchIdleReactivateCount;
168168

169+
private long starTreeQueryCount;
170+
private long starTreeQueryTimeInMillis;
171+
169172
@Nullable
170173
private RequestStatsLongHolder requestStatsLongHolder;
171174

@@ -177,6 +180,7 @@ private Stats() {
177180
// for internal use, initializes all counts to 0
178181
}
179182

183+
// Keeping this constructor for backward compatibility
180184
public Stats(
181185
long queryCount,
182186
long queryTimeInMillis,
@@ -198,6 +202,56 @@ public Stats(
198202
long suggestTimeInMillis,
199203
long suggestCurrent,
200204
long searchIdleReactivateCount
205+
) {
206+
this(
207+
queryCount,
208+
queryTimeInMillis,
209+
queryCurrent,
210+
concurrentQueryCount,
211+
concurrentQueryTimeInMillis,
212+
concurrentQueryCurrent,
213+
queryConcurrency,
214+
fetchCount,
215+
fetchTimeInMillis,
216+
fetchCurrent,
217+
scrollCount,
218+
scrollTimeInMillis,
219+
scrollCurrent,
220+
pitCount,
221+
pitTimeInMillis,
222+
pitCurrent,
223+
suggestCount,
224+
suggestTimeInMillis,
225+
suggestCurrent,
226+
searchIdleReactivateCount,
227+
0L, // starTreeQueryCount
228+
0L // starTreeQueryTimeInMillis
229+
);
230+
}
231+
232+
public Stats(
233+
long queryCount,
234+
long queryTimeInMillis,
235+
long queryCurrent,
236+
long concurrentQueryCount,
237+
long concurrentQueryTimeInMillis,
238+
long concurrentQueryCurrent,
239+
long queryConcurrency,
240+
long fetchCount,
241+
long fetchTimeInMillis,
242+
long fetchCurrent,
243+
long scrollCount,
244+
long scrollTimeInMillis,
245+
long scrollCurrent,
246+
long pitCount,
247+
long pitTimeInMillis,
248+
long pitCurrent,
249+
long suggestCount,
250+
long suggestTimeInMillis,
251+
long suggestCurrent,
252+
long searchIdleReactivateCount,
253+
long starTreeQueryCount,
254+
long starTreeQueryTimeInMillis
201255
) {
202256
this.requestStatsLongHolder = new RequestStatsLongHolder();
203257
this.queryCount = queryCount;
@@ -226,6 +280,9 @@ public Stats(
226280
this.pitCurrent = pitCurrent;
227281

228282
this.searchIdleReactivateCount = searchIdleReactivateCount;
283+
284+
this.starTreeQueryCount = starTreeQueryCount;
285+
this.starTreeQueryTimeInMillis = starTreeQueryTimeInMillis;
229286
}
230287

231288
private Stats(StreamInput in) throws IOException {
@@ -265,6 +322,11 @@ private Stats(StreamInput in) throws IOException {
265322
if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
266323
searchIdleReactivateCount = in.readVLong();
267324
}
325+
326+
if (in.getVersion().onOrAfter(Version.V_3_1_0)) {
327+
starTreeQueryCount = in.readVLong();
328+
starTreeQueryTimeInMillis = in.readVLong();
329+
}
268330
}
269331

270332
public void add(Stats stats) {
@@ -294,6 +356,9 @@ public void add(Stats stats) {
294356
pitCurrent += stats.pitCurrent;
295357

296358
searchIdleReactivateCount += stats.searchIdleReactivateCount;
359+
360+
starTreeQueryCount += stats.starTreeQueryCount;
361+
starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
297362
}
298363

299364
public void addForClosingShard(Stats stats) {
@@ -320,6 +385,9 @@ public void addForClosingShard(Stats stats) {
320385
queryConcurrency += stats.queryConcurrency;
321386

322387
searchIdleReactivateCount += stats.searchIdleReactivateCount;
388+
389+
starTreeQueryCount += stats.starTreeQueryCount;
390+
starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
323391
}
324392

325393
public long getQueryCount() {
@@ -430,6 +498,18 @@ public long getSearchIdleReactivateCount() {
430498
return searchIdleReactivateCount;
431499
}
432500

501+
public long getStarTreeQueryCount() {
502+
return starTreeQueryCount;
503+
}
504+
505+
public TimeValue getStarTreeQueryTime() {
506+
return new TimeValue(starTreeQueryTimeInMillis);
507+
}
508+
509+
public long getStarTreeQueryTimeInMillis() {
510+
return starTreeQueryTimeInMillis;
511+
}
512+
433513
public static Stats readStats(StreamInput in) throws IOException {
434514
return new Stats(in);
435515
}
@@ -479,6 +559,11 @@ public void writeTo(StreamOutput out) throws IOException {
479559
if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
480560
out.writeVLong(searchIdleReactivateCount);
481561
}
562+
563+
if (out.getVersion().onOrAfter(Version.V_3_1_0)) {
564+
out.writeVLong(starTreeQueryCount);
565+
out.writeVLong(starTreeQueryTimeInMillis);
566+
}
482567
}
483568

484569
@Override
@@ -492,6 +577,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
492577
builder.field(Fields.CONCURRENT_QUERY_CURRENT, concurrentQueryCurrent);
493578
builder.field(Fields.CONCURRENT_AVG_SLICE_COUNT, getConcurrentAvgSliceCount());
494579

580+
builder.field(Fields.STARTREE_QUERY_TOTAL, starTreeQueryCount);
581+
builder.humanReadableField(Fields.STARTREE_QUERY_TIME_IN_MILLIS, Fields.STARTREE_QUERY_TIME, getStarTreeQueryTime());
582+
495583
builder.field(Fields.FETCH_TOTAL, fetchCount);
496584
builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime());
497585
builder.field(Fields.FETCH_CURRENT, fetchCurrent);
@@ -677,6 +765,9 @@ static final class Fields {
677765
static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis";
678766
static final String CONCURRENT_QUERY_CURRENT = "concurrent_query_current";
679767
static final String CONCURRENT_AVG_SLICE_COUNT = "concurrent_avg_slice_count";
768+
static final String STARTREE_QUERY_TOTAL = "startree_query_total";
769+
static final String STARTREE_QUERY_TIME = "startree_query_time";
770+
static final String STARTREE_QUERY_TIME_IN_MILLIS = "startree_query_time_in_millis";
680771
static final String FETCH_TOTAL = "fetch_total";
681772
static final String FETCH_TIME = "fetch_time";
682773
static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis";

‎server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,9 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
133133
assert searchContext.searcher().getSlices() != null;
134134
statsHolder.queryConcurrencyMetric.inc(searchContext.searcher().getSlices().length);
135135
}
136+
if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) {
137+
statsHolder.starTreeQueryMetric.inc(tookInNanos);
138+
}
136139
}
137140
});
138141
}
@@ -245,6 +248,7 @@ static final class StatsHolder {
245248
final CounterMetric pitCurrent = new CounterMetric();
246249
final CounterMetric suggestCurrent = new CounterMetric();
247250
final CounterMetric searchIdleMetric = new CounterMetric();
251+
final MeanMetric starTreeQueryMetric = new MeanMetric();
248252

249253
SearchStats.Stats stats() {
250254
return new SearchStats.Stats(
@@ -267,7 +271,9 @@ SearchStats.Stats stats() {
267271
suggestMetric.count(),
268272
TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()),
269273
suggestCurrent.count(),
270-
searchIdleMetric.count()
274+
searchIdleMetric.count(),
275+
starTreeQueryMetric.count(),
276+
TimeUnit.NANOSECONDS.toMillis(starTreeQueryMetric.sum())
271277
);
272278
}
273279
}

‎server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -677,6 +677,18 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa
677677
);
678678
table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency");
679679

680+
table.addCell(
681+
"search.startree_query_total",
682+
"sibling:pri;alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
683+
);
684+
table.addCell("pri.startree.query_total", "default:false;text-align:right;desc:total star tree resolved queries");
685+
686+
table.addCell(
687+
"search.startree_query_time",
688+
"sibling:pri;alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
689+
);
690+
table.addCell("pri.startree.query_time", "default:false;text-align:right;desc:time spent in star tree queries");
691+
680692
table.addCell(
681693
"search.scroll_current",
682694
"sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts"
@@ -1011,6 +1023,12 @@ protected Table buildTable(
10111023
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount());
10121024
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount());
10131025

1026+
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryCount());
1027+
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryCount());
1028+
1029+
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryTime());
1030+
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryTime());
1031+
10141032
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent());
10151033
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent());
10161034

‎server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,15 @@ protected Table getTableWithHeader(final RestRequest request) {
340340
"alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts"
341341
);
342342

343+
table.addCell(
344+
"search.startree_query_total",
345+
"alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
346+
);
347+
table.addCell(
348+
"search.startree_query_time",
349+
"alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
350+
);
351+
343352
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
344353
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
345354
table.addCell(
@@ -556,6 +565,8 @@ Table buildTable(
556565
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCurrent());
557566
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitTime());
558567
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCount());
568+
table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryCount());
569+
table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryTime());
559570

560571
SegmentsStats segmentsStats = indicesStats == null ? null : indicesStats.getSegments();
561572
table.addCell(segmentsStats == null ? null : segmentsStats.getCount());

‎server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,15 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa
249249
"search.concurrent_avg_slice_count",
250250
"alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency"
251251
);
252+
table.addCell(
253+
"search.startree_query_total",
254+
"alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
255+
);
256+
table.addCell(
257+
"search.startree_query_time",
258+
"alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
259+
);
260+
252261
table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts");
253262
table.addCell(
254263
"search.scroll_time",
@@ -449,6 +458,9 @@ Table buildTable(
449458
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCount()));
450459
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentAvgSliceCount()));
451460

461+
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getStarTreeQueryCount()));
462+
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getStarTreeQueryTime()));
463+
452464
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
453465
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
454466
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));

‎server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,21 @@
5353

5454
public class SearchStatsTests extends OpenSearchTestCase implements SearchRequestOperationsListenerSupport {
5555

56-
// https://github.com/elastic/elasticsearch/issues/7644
57-
public void testShardLevelSearchGroupStats() throws Exception {
56+
public void testShardLevelSearchGroupStats() {
5857
// let's create two dummy search stats with groups
5958
Map<String, Stats> groupStats1 = new HashMap<>();
6059
Map<String, Stats> groupStats2 = new HashMap<>();
61-
groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1));
62-
SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1);
63-
SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2);
60+
groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1));
61+
SearchStats searchStats1 = new SearchStats(
62+
new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
63+
0,
64+
groupStats1
65+
);
66+
SearchStats searchStats2 = new SearchStats(
67+
new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
68+
0,
69+
groupStats2
70+
);
6471

6572
// adding these two search stats and checking group stats are correct
6673
searchStats1.add(searchStats2);
@@ -117,6 +124,8 @@ private static void assertStats(Stats stats, long equalTo) {
117124
assertEquals(equalTo, stats.getConcurrentQueryCount());
118125
assertEquals(equalTo, stats.getConcurrentQueryTimeInMillis());
119126
assertEquals(equalTo, stats.getConcurrentQueryCurrent());
127+
assertEquals(equalTo, stats.getStarTreeQueryCount());
128+
assertEquals(equalTo, stats.getStarTreeQueryTimeInMillis());
120129
assertEquals(equalTo, stats.getFetchCount());
121130
assertEquals(equalTo, stats.getFetchTimeInMillis());
122131
assertEquals(equalTo, stats.getFetchCurrent());

‎server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ private void assertTable(Table table) {
158158
assertThat(headers.get(6).value, equalTo("ip"));
159159
assertThat(headers.get(7).value, equalTo("id"));
160160
assertThat(headers.get(8).value, equalTo("node"));
161-
assertThat(headers.get(79).value, equalTo("docs.deleted"));
161+
assertThat(headers.get(81).value, equalTo("docs.deleted"));
162162

163163
final List<List<Table.Cell>> rows = table.getRows();
164164
assertThat(rows.size(), equalTo(shardRoutings.size()));
@@ -174,9 +174,9 @@ private void assertTable(Table table) {
174174
assertThat(row.get(4).value, equalTo(shardStats.getStats().getDocs().getCount()));
175175
assertThat(row.get(6).value, equalTo(localNode.getHostAddress()));
176176
assertThat(row.get(7).value, equalTo(localNode.getId()));
177-
assertThat(row.get(77).value, equalTo(shardStats.getDataPath()));
178-
assertThat(row.get(78).value, equalTo(shardStats.getStatePath()));
179-
assertThat(row.get(79).value, equalTo(shardStats.getStats().getDocs().getDeleted()));
177+
assertThat(row.get(79).value, equalTo(shardStats.getDataPath()));
178+
assertThat(row.get(80).value, equalTo(shardStats.getStatePath()));
179+
assertThat(row.get(81).value, equalTo(shardStats.getStats().getDocs().getDeleted()));
180180
}
181181
}
182182
}

0 commit comments

Comments
 (0)