Skip to content

Commit adcdfd4

Browse files
924060929924060929
committed
[fix](nereids) fix Input slot(s) not in child's output (#58471)
fix `Input slot(s) not in child's output`, introduced by #57204 (cherry picked from commit b788842)
1 parent 9eb6f7c commit adcdfd4

File tree

3 files changed

+209
-37
lines changed

3 files changed

+209
-37
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) {
456456
}
457457
Pair<Boolean, List<Slot>> replaced = replaceExpressions(olapScan.getOutput(), false, true);
458458
if (replaced.first) {
459-
return olapScan.withPrunedTypeSlots(replaced.second);
459+
return olapScan.withCachedOutput(replaced.second);
460460
}
461461
return olapScan;
462462
}
@@ -718,7 +718,7 @@ private void tryRecordReplaceSlots(Plan plan, Object checkObj, Set<Integer> shou
718718
boolean shouldPrune = false;
719719
for (Slot slot : output) {
720720
int slotId = slot.getExprId().asInt();
721-
if (replacedDataTypes.containsKey(slotId)) {
721+
if (slot.getDataType() instanceof NestedColumnPrunable && replacedDataTypes.containsKey(slotId)) {
722722
shouldReplaceSlots.add(slotId);
723723
shouldPrune = true;
724724
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java‎

Lines changed: 46 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan,
8181
///////////////////////////////////////////////////////////////////////////
8282

8383
/**
84-
/**
84+
* /**
8585
* The select materialized index id to read data from.
8686
*/
8787
private final long selectedIndexId;
@@ -104,6 +104,11 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan,
104104
*/
105105
private final Map<Pair<Long, String>, Slot> cacheSlotWithSlotName;
106106

107+
/**
108+
* this is the cache output to overwrite the output, the priority is higher than cacheSlotWithSlotName
109+
*/
110+
private final Optional<List<Slot>> cachedOutput;
111+
107112
///////////////////////////////////////////////////////////////////////////
108113
// Members for tablet ids.
109114
///////////////////////////////////////////////////////////////////////////
@@ -161,7 +166,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier) {
161166
table.getPartitionIds(), false,
162167
ImmutableList.of(),
163168
-1, false, PreAggStatus.unset(), ImmutableList.of(), ImmutableList.of(),
164-
Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of(),
169+
Maps.newHashMap(), Optional.empty(), Optional.empty(), false, ImmutableMap.of(),
165170
ImmutableList.of(), ImmutableList.of(), ImmutableList.of(),
166171
ImmutableList.of(), Optional.empty(), ImmutableList.of(), Optional.empty());
167172
}
@@ -170,7 +175,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, L
170175
List<String> hints, Optional<TableSample> tableSample, Collection<Slot> operativeSlots) {
171176
this(id, table, qualifier, Optional.empty(), Optional.empty(),
172177
table.getPartitionIds(), false, tabletIds,
173-
-1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(),
178+
-1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(), Optional.empty(),
174179
tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots,
175180
ImmutableList.of(), ImmutableList.of(), Optional.empty(), ImmutableList.of(), Optional.empty());
176181
}
@@ -183,7 +188,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, L
183188
this(id, table, qualifier, Optional.empty(), Optional.empty(),
184189
// must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1'
185190
specifiedPartitions, false, tabletIds,
186-
-1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(),
191+
-1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(), Optional.empty(),
187192
tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots,
188193
ImmutableList.of(), ImmutableList.of(), Optional.empty(),
189194
ImmutableList.of(), Optional.empty());
@@ -199,7 +204,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, L
199204
this(id, table, qualifier, Optional.empty(), Optional.empty(),
200205
selectedPartitionIds, false, tabletIds,
201206
selectedIndexId, true, preAggStatus,
202-
specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of(),
207+
specifiedPartitions, hints, Maps.newHashMap(), Optional.empty(), tableSample, true, ImmutableMap.of(),
203208
ImmutableList.of(), operativeSlots, ImmutableList.of(), ImmutableList.of(), Optional.empty(),
204209
ImmutableList.of(), Optional.empty());
205210
}
@@ -213,7 +218,7 @@ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
213218
List<Long> selectedTabletIds, long selectedIndexId, boolean indexSelected,
214219
PreAggStatus preAggStatus, List<Long> specifiedPartitions,
215220
List<String> hints, Map<Pair<Long, String>, Slot> cacheSlotWithSlotName,
216-
Optional<TableSample> tableSample, boolean directMvScan,
221+
Optional<List<Slot>> cachedOutput, Optional<TableSample> tableSample, boolean directMvScan,
217222
Map<String, Set<List<String>>> colToSubPathsMap, List<Long> specifiedTabletIds,
218223
Collection<Slot> operativeSlots, List<NamedExpression> virtualColumns,
219224
List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
@@ -245,6 +250,7 @@ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
245250
this.hints = Objects.requireNonNull(hints, "hints can not be null");
246251
this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName,
247252
"mvNameToSlot can not be null");
253+
this.cachedOutput = Objects.requireNonNull(cachedOutput, "cachedOutput can not be null");
248254
this.tableSample = tableSample;
249255
this.directMvScan = directMvScan;
250256
this.colToSubPathsMap = colToSubPathsMap;
@@ -337,8 +343,9 @@ public LogicalOlapScan withGroupExpression(Optional<GroupExpression> groupExpres
337343
groupExpression, Optional.of(getLogicalProperties()),
338344
selectedPartitionIds, partitionPruned, selectedTabletIds,
339345
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
340-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
341-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
346+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan,
347+
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
348+
scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
342349
}
343350

344351
@Override
@@ -347,8 +354,9 @@ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpr
347354
return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties,
348355
selectedPartitionIds, partitionPruned, selectedTabletIds,
349356
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
350-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
351-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
357+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan,
358+
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
359+
scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
352360
}
353361

354362
/**
@@ -359,8 +367,9 @@ public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds)
359367
Optional.empty(), Optional.of(getLogicalProperties()),
360368
selectedPartitionIds, true, selectedTabletIds,
361369
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
362-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
363-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
370+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan,
371+
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
372+
scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
364373
}
365374

366375
/**
@@ -373,7 +382,7 @@ public LogicalOlapScan withMaterializedIndexSelected(long indexId) {
373382
Optional.empty(), Optional.of(getLogicalProperties()),
374383
selectedPartitionIds, partitionPruned, selectedTabletIds,
375384
indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName,
376-
tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
385+
cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
377386
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
378387
}
379388

@@ -385,8 +394,9 @@ public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) {
385394
Optional.empty(), Optional.of(getLogicalProperties()),
386395
selectedPartitionIds, partitionPruned, selectedTabletIds,
387396
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
388-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
389-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
397+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan,
398+
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
399+
scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
390400
}
391401

392402
/**
@@ -397,8 +407,9 @@ public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) {
397407
Optional.empty(), Optional.of(getLogicalProperties()),
398408
selectedPartitionIds, partitionPruned, selectedTabletIds,
399409
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
400-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
401-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
410+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan,
411+
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
412+
scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
402413
}
403414

404415
/**
@@ -409,8 +420,9 @@ public LogicalOlapScan withColToSubPathsMap(Map<String, Set<List<String>>> colTo
409420
Optional.empty(), Optional.empty(),
410421
selectedPartitionIds, partitionPruned, selectedTabletIds,
411422
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
412-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
413-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
423+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan,
424+
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
425+
scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
414426
}
415427

416428
/**
@@ -421,8 +433,9 @@ public LogicalOlapScan withManuallySpecifiedTabletIds(List<Long> manuallySpecifi
421433
Optional.empty(), Optional.of(getLogicalProperties()),
422434
selectedPartitionIds, partitionPruned, selectedTabletIds,
423435
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
424-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds,
425-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
436+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan,
437+
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
438+
scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
426439
}
427440

428441
@Override
@@ -432,8 +445,9 @@ public LogicalOlapScan withRelationId(RelationId relationId) {
432445
Optional.empty(), Optional.empty(),
433446
selectedPartitionIds, false, selectedTabletIds,
434447
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
435-
hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap, selectedTabletIds,
436-
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, annOrderKeys, annLimit);
448+
hints, Maps.newHashMap(), Optional.empty(), tableSample, directMvScan,
449+
colToSubPathsMap, selectedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys,
450+
scoreLimit, annOrderKeys, annLimit);
437451
}
438452

439453
/**
@@ -451,7 +465,7 @@ public LogicalOlapScan withVirtualColumns(List<NamedExpression> virtualColumns)
451465
groupExpression, Optional.of(logicalProperties),
452466
selectedPartitionIds, partitionPruned, selectedTabletIds,
453467
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
454-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap,
468+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap,
455469
manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
456470
annOrderKeys, annLimit);
457471
}
@@ -475,7 +489,7 @@ public LogicalOlapScan withVirtualColumnsAndTopN(
475489
groupExpression, Optional.of(logicalProperties),
476490
selectedPartitionIds, partitionPruned, selectedTabletIds,
477491
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
478-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap,
492+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap,
479493
manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
480494
annOrderKeys, annLimit);
481495
}
@@ -523,6 +537,9 @@ public Optional<String> getSelectedMaterializedIndexName() {
523537

524538
@Override
525539
public List<Slot> computeOutput() {
540+
if (cachedOutput.isPresent()) {
541+
return cachedOutput.get();
542+
}
526543
if (selectedIndexId != ((OlapTable) table).getBaseIndexId()) {
527544
return getOutputByIndex(selectedIndexId);
528545
}
@@ -806,7 +823,7 @@ public CatalogRelation withOperativeSlots(Collection<Slot> operativeSlots) {
806823
groupExpression, Optional.of(getLogicalProperties()),
807824
selectedPartitionIds, partitionPruned, selectedTabletIds,
808825
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
809-
hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap,
826+
hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap,
810827
manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
811828
annOrderKeys, annLimit);
812829
}
@@ -841,19 +858,13 @@ private Map<Slot, Slot> constructReplaceMap(MTMV mtmv) {
841858
return replaceMap;
842859
}
843860

844-
/** withPrunedTypeSlots */
845-
public LogicalOlapScan withPrunedTypeSlots(List<Slot> outputSlots) {
846-
Map<Pair<Long, String>, Slot> replaceSlotMap = new HashMap<>();
847-
for (Slot outputSlot : outputSlots) {
848-
Pair<Long, String> key = Pair.of(selectedIndexId, outputSlot.getName());
849-
replaceSlotMap.put(key, outputSlot);
850-
}
851-
861+
/** withCachedOutput */
862+
public LogicalOlapScan withCachedOutput(List<Slot> outputSlots) {
852863
return new LogicalOlapScan(relationId, (Table) table, qualifier,
853864
groupExpression, Optional.empty(),
854865
selectedPartitionIds, partitionPruned, selectedTabletIds,
855866
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
856-
hints, replaceSlotMap, tableSample, directMvScan, colToSubPathsMap,
867+
hints, cacheSlotWithSlotName, Optional.of(outputSlots), tableSample, directMvScan, colToSubPathsMap,
857868
manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
858869
annOrderKeys, annLimit);
859870
}

0 commit comments

Comments
 (0)