diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java index 89868cf6fae6..ed6c58ad0fea 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseSingleStageBrokerRequestHandler.java @@ -842,8 +842,16 @@ protected BrokerResponse handleRequest(long requestId, String query, SqlNodeAndO if (QueryOptionsUtils.shouldDropResults(pinotQuery.getQueryOptions())) { brokerResponse.setResultTable(null); } - _brokerMetrics.addTimedTableValue(rawTableName, BrokerTimer.QUERY_TOTAL_TIME_MS, totalTimeMs, - TimeUnit.MILLISECONDS); + if (QueryOptionsUtils.isSecondaryWorkload(pinotQuery.getQueryOptions())) { + _brokerMetrics.addTimedTableValue(rawTableName, BrokerTimer.SECONDARY_WORKLOAD_QUERY_TOTAL_TIME_MS, totalTimeMs, + TimeUnit.MILLISECONDS); + _brokerMetrics.addTimedValue(BrokerTimer.SECONDARY_WORKLOAD_QUERY_TOTAL_TIME_MS, totalTimeMs, + TimeUnit.MILLISECONDS); + } else { + _brokerMetrics.addTimedTableValue(rawTableName, BrokerTimer.QUERY_TOTAL_TIME_MS, totalTimeMs, + TimeUnit.MILLISECONDS); + _brokerMetrics.addTimedValue(BrokerTimer.QUERY_TOTAL_TIME_MS, totalTimeMs, TimeUnit.MILLISECONDS); + } // Log query and stats _queryLogger.log( diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java index 349affe6c838..7ad9268b0d04 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandler.java @@ -41,6 +41,7 @@ import org.apache.pinot.common.request.BrokerRequest; import org.apache.pinot.common.response.broker.BrokerResponseNative; import org.apache.pinot.common.response.broker.QueryProcessingException; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.core.query.reduce.BrokerReduceService; import org.apache.pinot.core.transport.AsyncQueryResponse; import org.apache.pinot.core.transport.QueryResponse; @@ -116,7 +117,9 @@ protected BrokerResponseNative processBrokerRequest(long requestId, BrokerReques _failureDetector.notifyQuerySubmitted(asyncQueryResponse); Map finalResponses = asyncQueryResponse.getFinalResponses(); if (asyncQueryResponse.getStatus() == QueryResponse.Status.TIMED_OUT) { - _brokerMetrics.addMeteredTableValue(rawTableName, BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS, 1); + BrokerMeter meter = QueryOptionsUtils.isSecondaryWorkload(serverBrokerRequest.getPinotQuery().getQueryOptions()) + ? BrokerMeter.SECONDARY_WORKLOAD_BROKER_RESPONSES_WITH_TIMEOUTS : BrokerMeter.BROKER_RESPONSES_WITH_TIMEOUTS; + _brokerMetrics.addMeteredTableValue(rawTableName, meter, 1); } _failureDetector.notifyQueryFinished(asyncQueryResponse); _brokerMetrics.addPhaseTiming(rawTableName, BrokerQueryPhase.SCATTER_GATHER, @@ -162,7 +165,11 @@ protected BrokerResponseNative processBrokerRequest(long requestId, BrokerReques if (numServersNotResponded != 0) { brokerResponse.addException(new QueryProcessingException(QueryException.SERVER_NOT_RESPONDING_ERROR_CODE, String.format("%d servers %s not responded", numServersNotResponded, serversNotResponded))); - _brokerMetrics.addMeteredTableValue(rawTableName, BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1); + + BrokerMeter meter = QueryOptionsUtils.isSecondaryWorkload(serverBrokerRequest.getPinotQuery().getQueryOptions()) + ? BrokerMeter.SECONDARY_WORKLOAD_BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED + : BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED; + _brokerMetrics.addMeteredTableValue(rawTableName, meter, 1); } if (brokerResponse.getExceptionsSize() > 0) { _brokerMetrics.addMeteredTableValue(rawTableName, BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, 1); diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerMeter.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerMeter.java index c36b4ab504e6..ea6a66251ce8 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerMeter.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerMeter.java @@ -105,8 +105,12 @@ public enum BrokerMeter implements AbstractMetrics.Meter { // (numServersQueried > numServersResponded) BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED("badResponses", false), + SECONDARY_WORKLOAD_BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED("badResponses", false), + BROKER_RESPONSES_WITH_TIMEOUTS("badResponses", false), + SECONDARY_WORKLOAD_BROKER_RESPONSES_WITH_TIMEOUTS("badResponses", false), + // This metric track the number of broker responses with number of groups limit reached (potential bad responses). BROKER_RESPONSES_WITH_NUM_GROUPS_LIMIT_REACHED("badResponses", false), diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerTimer.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerTimer.java index 38c132eb4eaf..2e70cf69bb3f 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerTimer.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerTimer.java @@ -29,7 +29,9 @@ public enum BrokerTimer implements AbstractMetrics.Timer { ROUTING_TABLE_UPDATE_TIME(true), CLUSTER_CHANGE_QUEUE_TIME(true), // metric tracking the freshness lag for consuming segments FRESHNESS_LAG_MS(false), - QUERY_TOTAL_TIME_MS(false), + QUERY_TOTAL_TIME_MS(true), + + SECONDARY_WORKLOAD_QUERY_TOTAL_TIME_MS(true), // The latency of sending the request from broker to server NETTY_CONNECTION_SEND_REQUEST_LATENCY(false),