Search query failure and search star-tree query failure stats

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

Author: sandeshkr419

server/src/main/java/org/opensearch/index/search/stats/SearchStats.java

@@ -142,6 +142,7 @@ public static class Stats implements Writeable, ToXContentFragment {
         private long queryCount;
         private long queryTimeInMillis;
         private long queryCurrent;
+        private long queryFailedCount;
 
         private long concurrentQueryCount;
         private long concurrentQueryTimeInMillis;
@@ -169,6 +170,7 @@ public static class Stats implements Writeable, ToXContentFragment {
         private long starTreeQueryCount;
         private long starTreeQueryTimeInMillis;
         private long starTreeQueryCurrent;
+        private long starTreeQueryFailed;
 
         @Nullable
         private RequestStatsLongHolder requestStatsLongHolder;
@@ -191,6 +193,7 @@ private Stats(Builder builder) {
             this.queryCount = builder.queryCount;
             this.queryTimeInMillis = builder.queryTimeInMillis;
             this.queryCurrent = builder.queryCurrent;
+            this.queryFailedCount = builder.queryFailedCount;
 
             this.concurrentQueryCount = builder.concurrentQueryCount;
             this.concurrentQueryTimeInMillis = builder.concurrentQueryTimeInMillis;
@@ -218,6 +221,7 @@ private Stats(Builder builder) {
             this.starTreeQueryCount = builder.starTreeQueryCount;
             this.starTreeQueryTimeInMillis = builder.starTreeQueryTimeInMillis;
             this.starTreeQueryCurrent = builder.starTreeQueryCurrent;
+            this.starTreeQueryFailed = builder.starTreeQueryFailed;
         }
 
         /**
@@ -319,12 +323,18 @@ private Stats(StreamInput in) throws IOException {
                 starTreeQueryTimeInMillis = in.readVLong();
                 starTreeQueryCurrent = in.readVLong();
             }
+
+            if (in.getVersion().onOrAfter(Version.V_3_3_0)) {
+                queryFailedCount = in.readVLong();
+                starTreeQueryFailed = in.readVLong();
+            }
         }
 
         public void add(Stats stats) {
             queryCount += stats.queryCount;
             queryTimeInMillis += stats.queryTimeInMillis;
             queryCurrent += stats.queryCurrent;
+            queryFailedCount += stats.queryFailedCount;
 
             concurrentQueryCount += stats.concurrentQueryCount;
             concurrentQueryTimeInMillis += stats.concurrentQueryTimeInMillis;
@@ -352,11 +362,13 @@ public void add(Stats stats) {
             starTreeQueryCount += stats.starTreeQueryCount;
             starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
             starTreeQueryCurrent += stats.starTreeQueryCurrent;
+            starTreeQueryFailed += stats.starTreeQueryFailed;
         }
 
         public void addForClosingShard(Stats stats) {
             queryCount += stats.queryCount;
             queryTimeInMillis += stats.queryTimeInMillis;
+            queryFailedCount += stats.queryFailedCount;
 
             concurrentQueryCount += stats.concurrentQueryCount;
             concurrentQueryTimeInMillis += stats.concurrentQueryTimeInMillis;
@@ -381,6 +393,7 @@ public void addForClosingShard(Stats stats) {
 
             starTreeQueryCount += stats.starTreeQueryCount;
             starTreeQueryTimeInMillis += stats.starTreeQueryTimeInMillis;
+            starTreeQueryFailed += stats.starTreeQueryFailed;
         }
 
         public long getQueryCount() {
@@ -399,6 +412,10 @@ public long getQueryCurrent() {
             return queryCurrent;
         }
 
+        public long getQueryFailedCount() {
+            return queryFailedCount;
+        }
+
         public long getConcurrentQueryCount() {
             return concurrentQueryCount;
         }
@@ -507,6 +524,10 @@ public long getStarTreeQueryCurrent() {
             return starTreeQueryCurrent;
         }
 
+        public long getStarTreeQueryFailed() {
+            return starTreeQueryFailed;
+        }
+
         public static Stats readStats(StreamInput in) throws IOException {
             return new Stats(in);
         }
@@ -562,13 +583,19 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeVLong(starTreeQueryTimeInMillis);
                 out.writeVLong(starTreeQueryCurrent);
             }
+
+            if (out.getVersion().onOrAfter(Version.V_3_3_0)) {
+                out.writeVLong(queryFailedCount);
+                out.writeVLong(starTreeQueryFailed);
+            }
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(Fields.QUERY_TOTAL, queryCount);
             builder.humanReadableField(Fields.QUERY_TIME_IN_MILLIS, Fields.QUERY_TIME, getQueryTime());
             builder.field(Fields.QUERY_CURRENT, queryCurrent);
+            builder.field(Fields.QUERY_FAILED_TOTAL, queryFailedCount);
 
             builder.field(Fields.CONCURRENT_QUERY_TOTAL, concurrentQueryCount);
             builder.humanReadableField(Fields.CONCURRENT_QUERY_TIME_IN_MILLIS, Fields.CONCURRENT_QUERY_TIME, getConcurrentQueryTime());
@@ -578,6 +605,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.STARTREE_QUERY_TOTAL, starTreeQueryCount);
             builder.humanReadableField(Fields.STARTREE_QUERY_TIME_IN_MILLIS, Fields.STARTREE_QUERY_TIME, getStarTreeQueryTime());
             builder.field(Fields.STARTREE_QUERY_CURRENT, getStarTreeQueryCurrent());
+            builder.field(Fields.STARTREE_QUERY_FAILED, getStarTreeQueryFailed());
 
             builder.field(Fields.FETCH_TOTAL, fetchCount);
             builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime());
@@ -633,6 +661,7 @@ public static class Builder {
             private long queryCount = 0;
             private long queryTimeInMillis = 0;
             private long queryCurrent = 0;
+            private long queryFailedCount = 0;
             private long concurrentQueryCount = 0;
             private long concurrentQueryTimeInMillis = 0;
             private long concurrentQueryCurrent = 0;
@@ -653,6 +682,7 @@ public static class Builder {
             private long starTreeQueryCount = 0;
             private long starTreeQueryTimeInMillis = 0;
             private long starTreeQueryCurrent = 0;
+            private long starTreeQueryFailed = 0;
             @Nullable
             private RequestStatsLongHolder requestStatsLongHolder = null;
 
@@ -673,6 +703,11 @@ public Builder queryCurrent(long current) {
                 return this;
             }
 
+            public Builder queryFailed(long count) {
+                this.queryFailedCount = count;
+                return this;
+            }
+
             public Builder concurrentQueryCount(long count) {
                 this.concurrentQueryCount = count;
                 return this;
@@ -773,6 +808,11 @@ public Builder starTreeQueryCurrent(long current) {
                 return this;
             }
 
+            public Builder starTreeQueryFailed(long count) {
+                this.starTreeQueryFailed = count;
+                return this;
+            }
+
             /**
              * Creates a {@link Stats} object from the builder's current state.
              * @return A new Stats instance.
@@ -916,6 +956,7 @@ static final class Fields {
         static final String QUERY_TIME = "query_time";
         static final String QUERY_TIME_IN_MILLIS = "query_time_in_millis";
         static final String QUERY_CURRENT = "query_current";
+        static final String QUERY_FAILED_TOTAL = "query_failed";
         static final String CONCURRENT_QUERY_TOTAL = "concurrent_query_total";
         static final String CONCURRENT_QUERY_TIME = "concurrent_query_time";
         static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis";
@@ -925,6 +966,7 @@ static final class Fields {
         static final String STARTREE_QUERY_TIME = "startree_query_time";
         static final String STARTREE_QUERY_TIME_IN_MILLIS = "startree_query_time_in_millis";
         static final String STARTREE_QUERY_CURRENT = "startree_query_current";
+        static final String STARTREE_QUERY_FAILED = "startree_query_failed";
         static final String FETCH_TOTAL = "fetch_total";
         static final String FETCH_TIME = "fetch_time";
         static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis";

server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java

@@ -104,6 +104,7 @@ public void onPreQueryPhase(SearchContext searchContext) {
     @Override
     public void onFailedQueryPhase(SearchContext searchContext) {
         computeStats(searchContext, statsHolder -> {
+            statsHolder.queryFailed.inc();
             if (searchContext.hasOnlySuggest()) {
                 statsHolder.suggestCurrent.dec();
                 assert statsHolder.suggestCurrent.count() >= 0;
@@ -115,6 +116,7 @@ public void onFailedQueryPhase(SearchContext searchContext) {
                     assert statsHolder.concurrentQueryCurrent.count() >= 0;
                 }
                 if (searchContext.getQueryShardContext().getStarTreeQueryContext() != null) {
+                    statsHolder.starTreeQueryFailed.inc();
                     statsHolder.starTreeCurrent.dec();
                     assert statsHolder.starTreeCurrent.count() >= 0;
                 }
@@ -237,6 +239,7 @@ public void onSearchIdleReactivation() {
      */
     static final class StatsHolder {
         final MeanMetric queryMetric = new MeanMetric();
+        final CounterMetric queryFailed = new CounterMetric();
         final MeanMetric concurrentQueryMetric = new MeanMetric();
         final CounterMetric queryConcurrencyMetric = new CounterMetric();
         final MeanMetric fetchMetric = new MeanMetric();
@@ -259,11 +262,13 @@ static final class StatsHolder {
         final CounterMetric searchIdleMetric = new CounterMetric();
         final MeanMetric starTreeQueryMetric = new MeanMetric();
         final CounterMetric starTreeCurrent = new CounterMetric();
+        final CounterMetric starTreeQueryFailed = new CounterMetric();
 
         SearchStats.Stats stats() {
             return new SearchStats.Stats.Builder().queryCount(queryMetric.count())
                 .queryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()))
                 .queryCurrent(queryCurrent.count())
+                .queryFailed(queryFailed.count())
                 .concurrentQueryCount(concurrentQueryMetric.count())
                 .concurrentQueryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(concurrentQueryMetric.sum()))
                 .concurrentQueryCurrent(concurrentQueryCurrent.count())
@@ -284,6 +289,7 @@ SearchStats.Stats stats() {
                 .starTreeQueryCount(starTreeQueryMetric.count())
                 .starTreeQueryTimeInMillis(TimeUnit.NANOSECONDS.toMillis(starTreeQueryMetric.sum()))
                 .starTreeQueryCurrent(starTreeCurrent.count())
+                .starTreeQueryFailed(starTreeQueryFailed.count())
                 .build();
         }
     }

server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java

@@ -653,6 +653,13 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa
             "sibling:pri;alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops"
         );
         table.addCell("pri.search.query_total", "default:false;text-align:right;desc:total query phase ops");
+
+        table.addCell(
+                "search.query_failed",
+                "sibling:pri;alias:sqf,searchQueryFailed;default:false;text-align:right;desc:failed query phase ops"
+        );
+        table.addCell("pri.search.query_failed", "default:false;text-align:right;desc:failed query phase ops");
+
         table.addCell(
             "search.concurrent_query_current",
             "sibling:pri;alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops"
@@ -1017,6 +1024,9 @@ protected Table buildTable(
             table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount());
             table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount());
 
+            table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryFailedCount());
+            table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryFailedCount());
+
             table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent());
             table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent());
 

server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java

@@ -304,6 +304,7 @@ protected Table getTableWithHeader(final RestRequest request) {
         table.addCell("search.query_current", "alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops");
         table.addCell("search.query_time", "alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase");
         table.addCell("search.query_total", "alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops");
+        table.addCell("search.query_failed", "alias:sqf,searchQueryFailed;default:false;text-align:right;desc:total failed query phase ops");
         table.addCell(
             "search.concurrent_query_current",
             "alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops"
@@ -561,6 +562,7 @@ Table buildTable(
             table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCurrent());
             table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryTime());
             table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCount());
+            table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryFailedCount());
             table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCurrent());
             table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryTime());
             table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCount());

server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java

@@ -233,6 +233,7 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa
         table.addCell("search.query_current", "alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops");
         table.addCell("search.query_time", "alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase");
         table.addCell("search.query_total", "alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops");
+        table.addCell("search.query_failed", "alias:sqf,searchQueryFailed;default:false;text-align:right;desc:failed query phase ops");
         table.addCell(
             "search.concurrent_query_current",
             "alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops"
@@ -458,6 +459,7 @@ Table buildTable(
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent()));
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime()));
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount()));
+            table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryFailedCount()));
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCurrent()));
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryTime()));
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCount()));