Skip to content

Commit 7431dfd

Browse files
sandeshkr419sandeshkr419
committed
startree node,indices,shards stats
Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>
1 parent 621421e commit 7431dfd

File tree

9 files changed

+114
-13
lines changed

9 files changed

+114
-13
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2020
- Add functionality for plugins to inject QueryCollectorContext during QueryPhase ([#18637](https://github.com/opensearch-project/OpenSearch/pull/18637))
2121
- Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460))
2222
- Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530))
23-
- Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
23+
- Add Semantic Version field type mapper and extensive unit tests ([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
24+
- [Star-Tree] Add star-tree search related stats ([#18707](https://github.com/opensearch-project/OpenSearch/pull/18707))
2425

2526
### Changed
2627
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"Help":
22
- skip:
33
version: " - 2.13.99"
4-
reason: search idle reactivate count total is only added in 3.0.0
4+
reason: star-tree search stats is only added in 3.1.0
55
features: node_selector
66
- do:
77
cat.shards:
@@ -70,6 +70,8 @@
7070
search.concurrent_query_time .+ \n
7171
search.concurrent_query_total .+ \n
7272
search.concurrent_avg_slice_count .+ \n
73+
search.startree_query_total .+ \n
74+
search.startree_query_time_in_millis .+ \n
7375
search.scroll_current .+ \n
7476
search.scroll_time .+ \n
7577
search.scroll_total .+ \n

‎server/src/main/java/org/opensearch/index/search/stats/SearchStats.java‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,9 @@ public static class Stats implements Writeable, ToXContentFragment {
166166

167167
private long searchIdleReactivateCount;
168168

169+
private long starTreeQueryCount;
170+
private long starTreeQueryTimeInMillis;
171+
169172
@Nullable
170173
private RequestStatsLongHolder requestStatsLongHolder;
171174

@@ -197,7 +200,9 @@ public Stats(
197200
long suggestCount,
198201
long suggestTimeInMillis,
199202
long suggestCurrent,
200-
long searchIdleReactivateCount
203+
long searchIdleReactivateCount,
204+
long starTreeQueryCount,
205+
long starTreeQueryTimeInMillis
201206
) {
202207
this.requestStatsLongHolder = new RequestStatsLongHolder();
203208
this.queryCount = queryCount;
@@ -226,6 +231,9 @@ public Stats(
226231
this.pitCurrent = pitCurrent;
227232

228233
this.searchIdleReactivateCount = searchIdleReactivateCount;
234+
235+
this.starTreeQueryCount = starTreeQueryCount;
236+
this.starTreeQueryTimeInMillis = starTreeQueryTimeInMillis;
229237
}
230238

231239
private Stats(StreamInput in) throws IOException {
@@ -265,6 +273,11 @@ private Stats(StreamInput in) throws IOException {
265273
if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
266274
searchIdleReactivateCount = in.readVLong();
267275
}
276+
277+
if (in.getVersion().onOrAfter(Version.V_3_1_0)) {
278+
starTreeQueryCount = in.readVLong();
279+
starTreeQueryTimeInMillis = in.readVLong();
280+
}
268281
}
269282

270283
public void add(Stats stats) {
@@ -294,6 +307,9 @@ public void add(Stats stats) {
294307
pitCurrent += stats.pitCurrent;
295308

296309
searchIdleReactivateCount += stats.searchIdleReactivateCount;
310+
311+
starTreeQueryCount += stats.starTreeQueryCount;
312+
starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
297313
}
298314

299315
public void addForClosingShard(Stats stats) {
@@ -320,6 +336,9 @@ public void addForClosingShard(Stats stats) {
320336
queryConcurrency += stats.queryConcurrency;
321337

322338
searchIdleReactivateCount += stats.searchIdleReactivateCount;
339+
340+
starTreeQueryCount += stats.starTreeQueryCount;
341+
starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
323342
}
324343

325344
public long getQueryCount() {
@@ -430,6 +449,18 @@ public long getSearchIdleReactivateCount() {
430449
return searchIdleReactivateCount;
431450
}
432451

452+
public long getStarTreeQueryCount() {
453+
return starTreeQueryCount;
454+
}
455+
456+
public TimeValue getStarTreeQueryTime() {
457+
return new TimeValue(starTreeQueryTimeInMillis);
458+
}
459+
460+
public long getStarTreeQueryTimeInMillis() {
461+
return starTreeQueryTimeInMillis;
462+
}
463+
433464
public static Stats readStats(StreamInput in) throws IOException {
434465
return new Stats(in);
435466
}
@@ -479,6 +510,11 @@ public void writeTo(StreamOutput out) throws IOException {
479510
if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
480511
out.writeVLong(searchIdleReactivateCount);
481512
}
513+
514+
if (out.getVersion().onOrAfter(Version.V_3_1_0)) {
515+
out.writeVLong(starTreeQueryCount);
516+
out.writeVLong(starTreeQueryTimeInMillis);
517+
}
482518
}
483519

484520
@Override
@@ -492,6 +528,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
492528
builder.field(Fields.CONCURRENT_QUERY_CURRENT, concurrentQueryCurrent);
493529
builder.field(Fields.CONCURRENT_AVG_SLICE_COUNT, getConcurrentAvgSliceCount());
494530

531+
builder.field(Fields.STARTREE_QUERY_TOTAL, starTreeQueryCount);
532+
builder.humanReadableField(Fields.STARTREE_QUERY_TIME_IN_MILLIS, Fields.STARTREE_QUERY_TIME, getStarTreeQueryTime());
533+
495534
builder.field(Fields.FETCH_TOTAL, fetchCount);
496535
builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime());
497536
builder.field(Fields.FETCH_CURRENT, fetchCurrent);
@@ -677,6 +716,9 @@ static final class Fields {
677716
static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis";
678717
static final String CONCURRENT_QUERY_CURRENT = "concurrent_query_current";
679718
static final String CONCURRENT_AVG_SLICE_COUNT = "concurrent_avg_slice_count";
719+
static final String STARTREE_QUERY_TOTAL = "startree_query_total";
720+
static final String STARTREE_QUERY_TIME = "startree_query_time";
721+
static final String STARTREE_QUERY_TIME_IN_MILLIS = "startree_query_time_in_millis";
680722
static final String FETCH_TOTAL = "fetch_total";
681723
static final String FETCH_TIME = "fetch_time";
682724
static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis";

‎server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,9 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
133133
assert searchContext.searcher().getSlices() != null;
134134
statsHolder.queryConcurrencyMetric.inc(searchContext.searcher().getSlices().length);
135135
}
136+
if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) {
137+
statsHolder.starTreeQueryMetric.inc(tookInNanos);
138+
}
136139
}
137140
});
138141
}
@@ -245,6 +248,7 @@ static final class StatsHolder {
245248
final CounterMetric pitCurrent = new CounterMetric();
246249
final CounterMetric suggestCurrent = new CounterMetric();
247250
final CounterMetric searchIdleMetric = new CounterMetric();
251+
final MeanMetric starTreeQueryMetric = new MeanMetric();
248252

249253
SearchStats.Stats stats() {
250254
return new SearchStats.Stats(
@@ -267,7 +271,9 @@ SearchStats.Stats stats() {
267271
suggestMetric.count(),
268272
TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()),
269273
suggestCurrent.count(),
270-
searchIdleMetric.count()
274+
searchIdleMetric.count(),
275+
starTreeQueryMetric.count(),
276+
TimeUnit.NANOSECONDS.toMillis(starTreeQueryMetric.sum())
271277
);
272278
}
273279
}

‎server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -677,6 +677,18 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa
677677
);
678678
table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency");
679679

680+
table.addCell(
681+
"search.startree_query_total",
682+
"sibling:pri;alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
683+
);
684+
table.addCell("pri.startree.query_total", "default:false;text-align:right;desc:total star tree resolved queries");
685+
686+
table.addCell(
687+
"search.startree_query_time",
688+
"sibling:pri;alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
689+
);
690+
table.addCell("pri.startree.query_time", "default:false;text-align:right;desc:time spent in star tree queries");
691+
680692
table.addCell(
681693
"search.scroll_current",
682694
"sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts"
@@ -1011,6 +1023,12 @@ protected Table buildTable(
10111023
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount());
10121024
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount());
10131025

1026+
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryCount());
1027+
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryCount());
1028+
1029+
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getStarTreeQueryTime());
1030+
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getStarTreeQueryTime());
1031+
10141032
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent());
10151033
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent());
10161034

‎server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,15 @@ protected Table getTableWithHeader(final RestRequest request) {
340340
"alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts"
341341
);
342342

343+
table.addCell(
344+
"search.startree_query_total",
345+
"alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
346+
);
347+
table.addCell(
348+
"search.startree_query_time",
349+
"alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
350+
);
351+
343352
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
344353
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
345354
table.addCell(
@@ -556,6 +565,8 @@ Table buildTable(
556565
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCurrent());
557566
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitTime());
558567
table.addCell(searchStats == null ? null : searchStats.getTotal().getPitCount());
568+
table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryCount());
569+
table.addCell(searchStats == null ? null : searchStats.getTotal().getStarTreeQueryTime());
559570

560571
SegmentsStats segmentsStats = indicesStats == null ? null : indicesStats.getSegments();
561572
table.addCell(segmentsStats == null ? null : segmentsStats.getCount());

‎server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,15 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa
249249
"search.concurrent_avg_slice_count",
250250
"alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency"
251251
);
252+
table.addCell(
253+
"search.startree_query_total",
254+
"alias:stqc,startreeQueryTotal;default:false;text-align:right;desc:total star tree resolved queries"
255+
);
256+
table.addCell(
257+
"search.startree_query_time",
258+
"alias:stqt,startreeQueryTime;default:false;text-align:right;desc:time spent in star tree queries"
259+
);
260+
252261
table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts");
253262
table.addCell(
254263
"search.scroll_time",
@@ -449,6 +458,9 @@ Table buildTable(
449458
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCount()));
450459
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentAvgSliceCount()));
451460

461+
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getStarTreeQueryCount()));
462+
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getStarTreeQueryTime()));
463+
452464
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
453465
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
454466
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));

‎server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,21 @@
5353

5454
public class SearchStatsTests extends OpenSearchTestCase implements SearchRequestOperationsListenerSupport {
5555

56-
// https://github.com/elastic/elasticsearch/issues/7644
57-
public void testShardLevelSearchGroupStats() throws Exception {
56+
public void testShardLevelSearchGroupStats() {
5857
// let's create two dummy search stats with groups
5958
Map<String, Stats> groupStats1 = new HashMap<>();
6059
Map<String, Stats> groupStats2 = new HashMap<>();
61-
groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1));
62-
SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1);
63-
SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2);
60+
groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1));
61+
SearchStats searchStats1 = new SearchStats(
62+
new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
63+
0,
64+
groupStats1
65+
);
66+
SearchStats searchStats2 = new SearchStats(
67+
new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
68+
0,
69+
groupStats2
70+
);
6471

6572
// adding these two search stats and checking group stats are correct
6673
searchStats1.add(searchStats2);
@@ -117,6 +124,8 @@ private static void assertStats(Stats stats, long equalTo) {
117124
assertEquals(equalTo, stats.getConcurrentQueryCount());
118125
assertEquals(equalTo, stats.getConcurrentQueryTimeInMillis());
119126
assertEquals(equalTo, stats.getConcurrentQueryCurrent());
127+
assertEquals(equalTo, stats.getStarTreeQueryCount());
128+
assertEquals(equalTo, stats.getStarTreeQueryTimeInMillis());
120129
assertEquals(equalTo, stats.getFetchCount());
121130
assertEquals(equalTo, stats.getFetchTimeInMillis());
122131
assertEquals(equalTo, stats.getFetchCurrent());

‎server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ private void assertTable(Table table) {
158158
assertThat(headers.get(6).value, equalTo("ip"));
159159
assertThat(headers.get(7).value, equalTo("id"));
160160
assertThat(headers.get(8).value, equalTo("node"));
161-
assertThat(headers.get(79).value, equalTo("docs.deleted"));
161+
assertThat(headers.get(81).value, equalTo("docs.deleted"));
162162

163163
final List<List<Table.Cell>> rows = table.getRows();
164164
assertThat(rows.size(), equalTo(shardRoutings.size()));
@@ -174,9 +174,9 @@ private void assertTable(Table table) {
174174
assertThat(row.get(4).value, equalTo(shardStats.getStats().getDocs().getCount()));
175175
assertThat(row.get(6).value, equalTo(localNode.getHostAddress()));
176176
assertThat(row.get(7).value, equalTo(localNode.getId()));
177-
assertThat(row.get(77).value, equalTo(shardStats.getDataPath()));
178-
assertThat(row.get(78).value, equalTo(shardStats.getStatePath()));
179-
assertThat(row.get(79).value, equalTo(shardStats.getStats().getDocs().getDeleted()));
177+
assertThat(row.get(79).value, equalTo(shardStats.getDataPath()));
178+
assertThat(row.get(80).value, equalTo(shardStats.getStatePath()));
179+
assertThat(row.get(81).value, equalTo(shardStats.getStats().getDocs().getDeleted()));
180180
}
181181
}
182182
}

0 commit comments

Comments
 (0)