Skip to content

Commit d966f1f

Browse files
sandeshkr419sandeshkr419
committed
startree node,indices,shards stats
Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>
1 parent c2f41cf commit d966f1f

File tree

9 files changed

+258
-12
lines changed

9 files changed

+258
-12
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2525
- Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
2626
- Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708))
2727
- Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697))
28+
- Add Semantic Version field type mapper and extensive unit tests ([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
29+
- [Star-Tree] Add star-tree search related stats ([#18707](https://github.com/opensearch-project/OpenSearch/pull/18707))
2830

2931
### Changed
3032
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml‎

Lines changed: 99 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,110 @@
11
"Help":
2+
- skip:
3+
version: " - 3.0.99"
4+
reason: star-tree search stats is only added in 3.1.0
5+
features: node_selector
6+
- do:
7+
cat.shards:
8+
help: true
9+
node_selector:
10+
version: "3.1.0 - "
11+
12+
- match:
13+
$body: |
14+
/^ index .+ \n
15+
shard .+ \n
16+
prirep .+ \n
17+
state .+ \n
18+
docs .+ \n
19+
store .+ \n
20+
ip .+ \n
21+
id .+ \n
22+
node .+ \n
23+
sync_id .+ \n
24+
unassigned.reason .+ \n
25+
unassigned.at .+ \n
26+
unassigned.for .+ \n
27+
unassigned.details .+ \n
28+
recoverysource.type .+ \n
29+
completion.size .+ \n
30+
fielddata.memory_size .+ \n
31+
fielddata.evictions .+ \n
32+
query_cache.memory_size .+ \n
33+
query_cache.evictions .+ \n
34+
flush.total .+ \n
35+
flush.total_time .+ \n
36+
get.current .+ \n
37+
get.time .+ \n
38+
get.total .+ \n
39+
get.exists_time .+ \n
40+
get.exists_total .+ \n
41+
get.missing_time .+ \n
42+
get.missing_total .+ \n
43+
indexing.delete_current .+ \n
44+
indexing.delete_time .+ \n
45+
indexing.delete_total .+ \n
46+
indexing.index_current .+ \n
47+
indexing.index_time .+ \n
48+
indexing.index_total .+ \n
49+
indexing.index_failed .+ \n
50+
merges.current .+ \n
51+
merges.current_docs .+ \n
52+
merges.current_size .+ \n
53+
merges.total .+ \n
54+
merges.total_docs .+ \n
55+
merges.total_size .+ \n
56+
merges.total_time .+ \n
57+
refresh.total .+ \n
58+
refresh.time .+ \n
59+
refresh.external_total .+ \n
60+
refresh.external_time .+ \n
61+
refresh.listeners .+ \n
62+
search.fetch_current .+ \n
63+
search.fetch_time .+ \n
64+
search.fetch_total .+ \n
65+
search.open_contexts .+ \n
66+
search.query_current .+ \n
67+
search.query_time .+ \n
68+
search.query_total .+ \n
69+
search.concurrent_query_current .+ \n
70+
search.concurrent_query_time .+ \n
71+
search.concurrent_query_total .+ \n
72+
search.concurrent_avg_slice_count .+ \n
73+
search.startree_query_total .+ \n
74+
search.startree_query_time_in_millis .+ \n
75+
search.scroll_current .+ \n
76+
search.scroll_time .+ \n
77+
search.scroll_total .+ \n
78+
search.point_in_time_current .+ \n
79+
search.point_in_time_time .+ \n
80+
search.point_in_time_total .+ \n
81+
search.search_idle_reactivate_count_total .+ \n
82+
segments.count .+ \n
83+
segments.memory .+ \n
84+
segments.index_writer_memory .+ \n
85+
segments.version_map_memory .+ \n
86+
segments.fixed_bitset_memory .+ \n
87+
seq_no.max .+ \n
88+
seq_no.local_checkpoint .+ \n
89+
seq_no.global_checkpoint .+ \n
90+
warmer.current .+ \n
91+
warmer.total .+ \n
92+
warmer.total_time .+ \n
93+
path.data .+ \n
94+
path.state .+ \n
95+
docs.deleted .+ \n
96+
$/
97+
---
98+
"Help from 2.14.0 to 3.0.99":
299
- skip:
3100
version: " - 2.13.99"
4-
reason: search idle reactivate count total is only added in 3.0.0
101+
reason: search idle reactivate count total is only added in 3.0.0
5102
features: node_selector
6103
- do:
7104
cat.shards:
8105
help: true
9106
node_selector:
10-
version: "2.14.0 - "
107+
version: "2.14.0 - 3.0.99"
11108

12109
- match:
13110
$body: |

‎server/src/main/java/org/opensearch/index/search/stats/SearchStats.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,9 @@ public static class Stats implements Writeable, ToXContentFragment {
166166

167167
private long searchIdleReactivateCount;
168168

169+
private long starTreeQueryCount;
170+
private long starTreeQueryTimeInMillis;
171+
169172
@Nullable
170173
private RequestStatsLongHolder requestStatsLongHolder;
171174

@@ -177,6 +180,7 @@ private Stats() {
177180
// for internal use, initializes all counts to 0
178181
}
179182

183+
// Keeping this constructor for backward compatibility
180184
public Stats(
181185
long queryCount,
182186
long queryTimeInMillis,
@@ -198,6 +202,56 @@ public Stats(
198202
long suggestTimeInMillis,
199203
long suggestCurrent,
200204
long searchIdleReactivateCount
205+
) {
206+
this(
207+
queryCount,
208+
queryTimeInMillis,
209+
queryCurrent,
210+
concurrentQueryCount,
211+
concurrentQueryTimeInMillis,
212+
concurrentQueryCurrent,
213+
queryConcurrency,
214+
fetchCount,
215+
fetchTimeInMillis,
216+
fetchCurrent,
217+
scrollCount,
218+
scrollTimeInMillis,
219+
scrollCurrent,
220+
pitCount,
221+
pitTimeInMillis,
222+
pitCurrent,
223+
suggestCount,
224+
suggestTimeInMillis,
225+
suggestCurrent,
226+
searchIdleReactivateCount,
227+
0L, // starTreeQueryCount
228+
0L // starTreeQueryTimeInMillis
229+
);
230+
}
231+
232+
public Stats(
233+
long queryCount,
234+
long queryTimeInMillis,
235+
long queryCurrent,
236+
long concurrentQueryCount,
237+
long concurrentQueryTimeInMillis,
238+
long concurrentQueryCurrent,
239+
long queryConcurrency,
240+
long fetchCount,
241+
long fetchTimeInMillis,
242+
long fetchCurrent,
243+
long scrollCount,
244+
long scrollTimeInMillis,
245+
long scrollCurrent,
246+
long pitCount,
247+
long pitTimeInMillis,
248+
long pitCurrent,
249+
long suggestCount,
250+
long suggestTimeInMillis,
251+
long suggestCurrent,
252+
long searchIdleReactivateCount,
253+
long starTreeQueryCount,
254+
long starTreeQueryTimeInMillis
201255
) {
202256
this.requestStatsLongHolder = new RequestStatsLongHolder();
203257
this.queryCount = queryCount;
@@ -226,6 +280,9 @@ public Stats(
226280
this.pitCurrent = pitCurrent;
227281

228282
this.searchIdleReactivateCount = searchIdleReactivateCount;
283+
284+
this.starTreeQueryCount = starTreeQueryCount;
285+
this.starTreeQueryTimeInMillis = starTreeQueryTimeInMillis;
229286
}
230287

231288
private Stats(StreamInput in) throws IOException {
@@ -265,6 +322,11 @@ private Stats(StreamInput in) throws IOException {
265322
if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
266323
searchIdleReactivateCount = in.readVLong();
267324
}
325+
326+
if (in.getVersion().onOrAfter(Version.V_3_1_0)) {
327+
starTreeQueryCount = in.readVLong();
328+
starTreeQueryTimeInMillis = in.readVLong();
329+
}
268330
}
269331

270332
public void add(Stats stats) {
@@ -294,6 +356,9 @@ public void add(Stats stats) {
294356
pitCurrent += stats.pitCurrent;
295357

296358
searchIdleReactivateCount += stats.searchIdleReactivateCount;
359+
360+
starTreeQueryCount += stats.starTreeQueryCount;
361+
starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
297362
}
298363

299364
public void addForClosingShard(Stats stats) {
@@ -320,6 +385,9 @@ public void addForClosingShard(Stats stats) {
320385
queryConcurrency += stats.queryConcurrency;
321386

322387
searchIdleReactivateCount += stats.searchIdleReactivateCount;
388+
389+
starTreeQueryCount += stats.starTreeQueryCount;
390+
starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
323391
}
324392

325393
public long getQueryCount() {
@@ -430,6 +498,18 @@ public long getSearchIdleReactivateCount() {
430498
return searchIdleReactivateCount;
431499
}
432500

501+
public long getStarTreeQueryCount() {
502+
return starTreeQueryCount;
503+
}
504+
505+
public TimeValue getStarTreeQueryTime() {
506+
return new TimeValue(starTreeQueryTimeInMillis);
507+
}
508+
509+
public long getStarTreeQueryTimeInMillis() {
510+
return starTreeQueryTimeInMillis;
511+
}
512+
433513
public static Stats readStats(StreamInput in) throws IOException {
434514
return new Stats(in);
435515
}
@@ -479,6 +559,11 @@ public void writeTo(StreamOutput out) throws IOException {
479559
if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
480560
out.writeVLong(searchIdleReactivateCount);
481561
}
562+
563+
if (out.getVersion().onOrAfter(Version.V_3_1_0)) {
564+
out.writeVLong(starTreeQueryCount);
565+
out.writeVLong(starTreeQueryTimeInMillis);
566+
}
482567
}
483568

484569
@Override
@@ -492,6 +577,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
492577
builder.field(Fields.CONCURRENT_QUERY_CURRENT, concurrentQueryCurrent);
493578
builder.field(Fields.CONCURRENT_AVG_SLICE_COUNT, getConcurrentAvgSliceCount());
494579

580+
builder.field(Fields.STARTREE_QUERY_TOTAL, starTreeQueryCount);
581+
builder.humanReadableField(Fields.STARTREE_QUERY_TIME_IN_MILLIS, Fields.STARTREE_QUERY_TIME, getStarTreeQueryTime());
582+
495583
builder.field(Fields.FETCH_TOTAL, fetchCount);
496584
builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime());
497585
builder.field(Fields.FETCH_CURRENT, fetchCurrent);
@@ -677,6 +765,9 @@ static final class Fields {
677765
static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis";
678766
static final String CONCURRENT_QUERY_CURRENT = "concurrent_query_current";
679767
static final String CONCURRENT_AVG_SLICE_COUNT = "concurrent_avg_slice_count";
768+
static final String STARTREE_QUERY_TOTAL = "startree_query_total";
769+
static final String STARTREE_QUERY_TIME = "startree_query_time";
770+
static final String STARTREE_QUERY_TIME_IN_MILLIS = "startree_query_time_in_millis";
680771
static final String FETCH_TOTAL = "fetch_total";
681772
static final String FETCH_TIME = "fetch_time";
682773
static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis";

‎server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,9 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
133133
assert searchContext.searcher().getSlices() != null;
134134
statsHolder.queryConcurrencyMetric.inc(searchContext.searcher().getSlices().length);
135135
}
136+
if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) {
137+
statsHolder.starTreeQueryMetric.inc(tookInNanos);
138+
}
136139
}
137140
});
138141
}
@@ -245,6 +248,7 @@ static final class StatsHolder {
245248
final CounterMetric pitCurrent = new CounterMetric();
246249
final CounterMetric suggestCurrent = new CounterMetric();
247250
final CounterMetric searchIdleMetric = new CounterMetric();
251+
final MeanMetric starTreeQueryMetric = new MeanMetric();
248252

249253
SearchStats.Stats stats() {
250254
return new SearchStats.Stats(
@@ -267,7 +271,9 @@ SearchStats.Stats stats() {
267271
suggestMetric.count(),
268272
TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()),
269273
suggestCurrent.count(),
270-
searchIdleMetric.count()
274+
searchIdleMetric.count(),
275+
starTreeQueryMetric.count(),
276+
TimeUnit.NANOSECONDS.toMillis(starTreeQueryMetric.sum())
271277
);
272278
}
273279
}

‎server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -677,6 +677,18 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa
677677
);
678678
table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency");
679679

680+
table.addCell(
681+
"search.startree_query_total",
682+
"sibling:pri;alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
683+
);
684+
table.addCell("pri.startree.query_total", "default:false;text-align:right;desc:total star tree resolved queries");
685+
686+
table.addCell(
687+
"search.startree_query_time",
688+
"sibling:pri;alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
689+
);
690+
table.addCell("pri.startree.query_time", "default:false;text-align:right;desc:time spent in star tree queries");
691+
680692
table.addCell(
681693
"search.scroll_current",
682694
"sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts"
@@ -1011,6 +1023,12 @@ protected Table buildTable(
10111023
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount());
10121024
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount());
10131025

1026+
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryCount());
1027+
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryCount());
1028+
1029+
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryTime());
1030+
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryTime());
1031+
10141032
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent());
10151033
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent());
10161034

‎server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,15 @@ protected Table getTableWithHeader(final RestRequest request) {
340340
"alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts"
341341
);
342342

343+
table.addCell(
344+
"search.startree_query_total",
345+
"alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
346+
);
347+
table.addCell(
348+
"search.startree_query_time",
349+
"alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
350+
);
351+
343352
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
344353
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
345354
table.addCell(
@@ -556,6 +565,8 @@ Table buildTable(
556565
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCurrent());
557566
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitTime());
558567
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCount());
568+
table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryCount());
569+
table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryTime());
559570

560571
SegmentsStats segmentsStats = indicesStats == null ? null : indicesStats.getSegments();
561572
table.addCell(segmentsStats == null ? null : segmentsStats.getCount());

0 commit comments

Comments
 (0)