From 34228a91cb93dceb8131f6de67eca1eb44f22e03 Mon Sep 17 00:00:00 2001 From: Jackie Tien Date: Sun, 24 Nov 2024 15:05:46 +0800 Subject: [PATCH] Fix some query metric --- .../operator/source/SeriesScanUtil.java | 6 +- .../metric/SeriesScanCostMetricSet.java | 22 +++--- .../plan/planner/TableOperatorGenerator.java | 75 +++++++++++++------ .../plan/relational/sql/parser/SqlParser.java | 8 ++ .../db/storageengine/buffer/ChunkCache.java | 8 +- 5 files changed, 83 insertions(+), 36 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 15fe07161a69..c46f1e9ca5bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -49,6 +49,7 @@ import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.page.AlignedPageReader; +import org.apache.tsfile.read.reader.page.TablePageReader; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.RamUsageEstimator; @@ -1228,7 +1229,10 @@ protected static class VersionPageReader { this.version = new MergeReaderPriority(fileTimestamp, version, offset, isSeq); this.data = data; this.isSeq = isSeq; - this.isAligned = data instanceof AlignedPageReader || data instanceof MemAlignedPageReader; + this.isAligned = + data instanceof AlignedPageReader + || data instanceof MemAlignedPageReader + || data instanceof TablePageReader; this.isMem = data instanceof MemPageReader || data instanceof MemAlignedPageReader; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java index ab6fb12f5963..ebaa7c030d9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java @@ -1107,28 +1107,28 @@ private void unbindConstructChunkReader(AbstractMetricService metricService) { ///////////////////////////////////////////////////////////////////////////////////////////////// private static final String READ_CHUNK = "read_chunk"; private static final String ALL = "all"; - public static final String READ_CHUNK_ALL = READ_CHUNK + "_" + ALL; + public static final String READ_CHUNK_CACHE = READ_CHUNK + "_" + CACHE; public static final String READ_CHUNK_FILE = READ_CHUNK + "_" + FILE; - private Timer readChunkAllTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer readChunkCacheTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private Timer readChunkFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER; private void bindReadChunk(AbstractMetricService metricService) { - readChunkAllTimer = + readChunkCacheTimer = metricService.getOrCreateTimer( Metric.SERIES_SCAN_COST.toString(), MetricLevel.IMPORTANT, Tag.STAGE.toString(), - READ_CHUNK, + READ_CHUNK_CACHE, Tag.TYPE.toString(), NULL, Tag.FROM.toString(), - ALL); + CACHE); readChunkFileTimer = metricService.getOrCreateTimer( Metric.SERIES_SCAN_COST.toString(), MetricLevel.IMPORTANT, Tag.STAGE.toString(), - READ_CHUNK, + READ_CHUNK_FILE, Tag.TYPE.toString(), NULL, Tag.FROM.toString(), @@ -1136,16 +1136,16 @@ private void bindReadChunk(AbstractMetricService metricService) { } private void unbindReadChunk(AbstractMetricService metricService) { - readChunkAllTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + readChunkCacheTimer = DoNothingMetricManager.DO_NOTHING_TIMER; readChunkFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER; - Arrays.asList(ALL, FILE) + Arrays.asList(CACHE, FILE) .forEach( from -> metricService.remove( MetricType.TIMER, Metric.SERIES_SCAN_COST.toString(), Tag.STAGE.toString(), - READ_CHUNK, + READ_CHUNK + "_" + from, Tag.TYPE.toString(), NULL, Tag.FROM.toString(), @@ -1571,8 +1571,8 @@ public void recordSeriesScanCost(String type, long cost) { case READ_TIMESERIES_METADATA_FILE: readTimeseriesMetadataFileTimer.updateNanos(cost); break; - case READ_CHUNK_ALL: - readChunkAllTimer.updateNanos(cost); + case READ_CHUNK_CACHE: + readChunkCacheTimer.updateNanos(cost); break; case READ_CHUNK_FILE: readChunkFileTimer.updateNanos(cost); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index fe0e8671f4af..adce0a3b6079 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -383,7 +383,7 @@ public Operator visitTableScan(TableScanNode node, LocalExecutionPlanContext con .addOperatorContext( context.getNextOperatorId(), node.getPlanNodeId(), - TableScanNode.class.getSimpleName()); + TableScanOperator.class.getSimpleName()); int maxTsBlockLineNum = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); if (context.getTypeProvider().getTemplatedInfo() != null) { @@ -1132,7 +1132,7 @@ public Operator visitStreamSort(StreamSortNode node, LocalExecutionPlanContext c .addOperatorContext( context.getNextOperatorId(), node.getPlanNodeId(), - StreamSortNode.class.getSimpleName()); + TableStreamSortOperator.class.getSimpleName()); List dataTypes = getOutputColumnTypes(node, context.getTypeProvider()); int sortItemsCount = node.getOrderingScheme().getOrderBy().size(); @@ -1176,11 +1176,6 @@ public Operator visitStreamSort(StreamSortNode node, LocalExecutionPlanContext c @Override public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { - OperatorContext operatorContext = - context - .getDriverContext() - .addOperatorContext( - context.getNextOperatorId(), node.getPlanNodeId(), JoinNode.class.getSimpleName()); List dataTypes = getOutputColumnTypes(node, context.getTypeProvider()); Operator leftChild = node.getLeftChild().accept(this, context); @@ -1221,6 +1216,13 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { } if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.INNER) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + TableInnerJoinOperator.class.getSimpleName()); return new TableInnerJoinOperator( operatorContext, leftChild, @@ -1232,6 +1234,13 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { ASC_TIME_COMPARATOR, dataTypes); } else if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.FULL) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + TableFullOuterJoinOperator.class.getSimpleName()); return new TableFullOuterJoinOperator( operatorContext, leftChild, @@ -1357,25 +1366,28 @@ public Operator visitTableDeviceQueryCount( @Override public Operator visitAggregation(AggregationNode node, LocalExecutionPlanContext context) { - OperatorContext operatorContext = - context - .getDriverContext() - .addOperatorContext( - context.getNextOperatorId(), - node.getPlanNodeId(), - AggregationNode.class.getSimpleName()); + Operator child = node.getChild().accept(this, context); if (node.getGroupingKeys().isEmpty()) { - return planGlobalAggregation(node, child, context.getTypeProvider(), operatorContext); + return planGlobalAggregation(node, child, context.getTypeProvider(), context); } - return planGroupByAggregation(node, child, context.getTypeProvider(), operatorContext); + return planGroupByAggregation(node, child, context.getTypeProvider(), context); } private Operator planGlobalAggregation( - AggregationNode node, Operator child, TypeProvider typeProvider, OperatorContext context) { - + AggregationNode node, + Operator child, + TypeProvider typeProvider, + LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + AggregationOperator.class.getSimpleName()); Map aggregationMap = node.getAggregations(); ImmutableList.Builder aggregatorBuilder = new ImmutableList.Builder<>(); Map childLayout = @@ -1393,7 +1405,7 @@ private Operator planGlobalAggregation( typeProvider, true, null))); - return new AggregationOperator(context, child, aggregatorBuilder.build()); + return new AggregationOperator(operatorContext, child, aggregatorBuilder.build()); } // timeColumnName will only be set for AggTableScan. @@ -1438,7 +1450,7 @@ private Operator planGroupByAggregation( AggregationNode node, Operator child, TypeProvider typeProvider, - OperatorContext operatorContext) { + LocalExecutionPlanContext context) { Map childLayout = makeLayoutFromOutputSymbols(node.getChild().getOutputSymbols()); @@ -1458,6 +1470,13 @@ private Operator planGroupByAggregation( buildAggregator( childLayout, k, v, node.getStep(), typeProvider, true, null))); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + StreamingAggregationOperator.class.getSimpleName()); return new StreamingAggregationOperator( operatorContext, child, @@ -1499,6 +1518,13 @@ private Operator planGroupByAggregation( } List preGroupedChannels = preGroupedChannelsBuilder.build(); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + StreamingHashAggregationOperator.class.getSimpleName()); return new StreamingHashAggregationOperator( operatorContext, child, @@ -1522,6 +1548,13 @@ private Operator planGroupByAggregation( (k, v) -> aggregatorBuilder.add( buildGroupByAggregator(childLayout, k, v, node.getStep(), typeProvider))); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + HashAggregationOperator.class.getSimpleName()); return new HashAggregationOperator( operatorContext, @@ -1739,7 +1772,7 @@ public Operator visitAggregationTableScan( .addOperatorContext( context.getNextOperatorId(), node.getPlanNodeId(), - AggregationTableScanNode.class.getSimpleName()); + TableAggregationTableScanOperator.class.getSimpleName()); SeriesScanOptions.Builder scanOptionsBuilder = node.getTimePredicate().isPresent() ? getSeriesScanOptionsBuilder(context, node.getTimePredicate().get()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java index b20fb8293269..1ee59aea6516 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.parser; +import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; @@ -54,6 +55,10 @@ import static java.util.Objects.requireNonNull; public class SqlParser { + + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); + private static final ANTLRErrorListener LEXER_ERROR_LISTENER = new BaseErrorListener() { @Override @@ -131,6 +136,7 @@ private Node invokeParser( Optional location, Function parseFunction, ZoneId zoneId) { + long startTime = System.nanoTime(); try { RelationalSqlLexer lexer = new RelationalSqlLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); @@ -190,6 +196,8 @@ public Token recoverInline(Parser recognizer) throws RecognitionException { return new AstBuilder(location.orElse(null), zoneId).visit(tree); } catch (StackOverflowError e) { throw new ParsingException(name + " is too large (stack overflow while parsing)"); + } finally { + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java index 5914272c7cc8..a604409035f6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java @@ -47,7 +47,7 @@ import java.util.function.Function; import java.util.function.LongConsumer; -import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_ALL; +import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_CACHE; import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_FILE; /** @@ -162,11 +162,13 @@ private Chunk get( } finally { if (chunkLoader.isCacheMiss()) { cacheMissAdder.accept(1); + SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( + READ_CHUNK_FILE, System.nanoTime() - startTime); } else { cacheHitAdder.accept(1); + SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( + READ_CHUNK_CACHE, System.nanoTime() - startTime); } - SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost( - READ_CHUNK_ALL, System.nanoTime() - startTime); } }