Skip to content

Commit f61d01c

Browse files
committed
[fix](nereids) fix prune map type cause backend core (#58573)
fix prune map type cause backend core, when the map type is changed, we should not prune the nested column type, introduced by #57204 (cherry picked from commit 1d7f6c4)
1 parent 9eb6f7c commit f61d01c

File tree

2 files changed

+55
-2
lines changed

2 files changed

+55
-2
lines changed

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,9 @@
5050
import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
5151
import org.apache.doris.nereids.trees.expressions.literal.Literal;
5252
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
53+
import org.apache.doris.nereids.types.ArrayType;
5354
import org.apache.doris.nereids.types.DataType;
55+
import org.apache.doris.nereids.types.MapType;
5456
import org.apache.doris.nereids.types.NestedColumnPrunable;
5557
import org.apache.doris.nereids.types.StructField;
5658
import org.apache.doris.nereids.types.StructType;
@@ -65,6 +67,7 @@
6567
import java.util.LinkedList;
6668
import java.util.List;
6769
import java.util.Map;
70+
import java.util.Map.Entry;
6871
import java.util.Objects;
6972
import java.util.Stack;
7073

@@ -135,7 +138,8 @@ public Void visitAlias(Alias alias, CollectorContext context) {
135138
public Void visitCast(Cast cast, CollectorContext context) {
136139
if (!context.accessPathBuilder.isEmpty()
137140
&& cast.getDataType() instanceof NestedColumnPrunable
138-
&& cast.child().getDataType() instanceof NestedColumnPrunable) {
141+
&& cast.child().getDataType() instanceof NestedColumnPrunable
142+
&& !mapTypeIsChanged(cast.child().getDataType(), cast.getDataType(), false)) {
139143

140144
DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
141145
DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
@@ -509,4 +513,46 @@ public int hashCode() {
509513
return path.hashCode();
510514
}
511515
}
516+
517+
// if the map type is changed, we can not prune the type, because the map type need distinct the keys,
518+
// e.g. select map_values(cast(map(3.0, 1, 3.1, 2) as map<int, int>));
519+
// the result is [2] because the keys: 3.0 and 3.1 will cast to 3 and the second entry remained.
520+
// backend will throw exception because it can not only access the values without the cast keys,
521+
// so we should check whether the map type is changed, if not changed, we can prune the type.
522+
private static boolean mapTypeIsChanged(DataType originType, DataType castType, boolean inMap) {
523+
if (originType.isMapType()) {
524+
MapType originMapType = (MapType) originType;
525+
MapType castMapType = (MapType) castType;
526+
if (mapTypeIsChanged(originMapType.getKeyType(), castMapType.getKeyType(), true)
527+
|| mapTypeIsChanged(originMapType.getValueType(), castMapType.getValueType(), true)) {
528+
return true;
529+
}
530+
return false;
531+
} else if (originType.isStructType()) {
532+
StructType originStructType = (StructType) originType;
533+
StructType castStructType = (StructType) castType;
534+
List<Entry<String, StructField>> originFields
535+
= new ArrayList<>(originStructType.getNameToFields().entrySet());
536+
List<Entry<String, StructField>> castFields
537+
= new ArrayList<>(castStructType.getNameToFields().entrySet());
538+
539+
for (int i = 0; i < originFields.size(); i++) {
540+
DataType originFieldType = originFields.get(i).getValue().getDataType();
541+
DataType castFieldType = castFields.get(i).getValue().getDataType();
542+
if (mapTypeIsChanged(originFieldType, castFieldType, inMap)) {
543+
return true;
544+
}
545+
}
546+
return false;
547+
} else if (originType.isArrayType()) {
548+
ArrayType originArrayType = (ArrayType) originType;
549+
ArrayType castArrayType = (ArrayType) castType;
550+
return mapTypeIsChanged(originArrayType.getItemType(), castArrayType.getItemType(), inMap);
551+
} else if (inMap) {
552+
return !originType.equals(castType);
553+
} else {
554+
// other type changed which not in map will not affect the map
555+
return false;
556+
}
557+
}
512558
}

fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,13 +140,20 @@ public void testStruct() throws Throwable {
140140

141141
@Test
142142
public void testPruneCast() throws Exception {
143+
// the map type is changed, so we can not prune type
144+
assertColumn("select struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:int>>>>), 'k') from tbl",
145+
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
146+
ImmutableList.of(path("s")),
147+
ImmutableList.of()
148+
);
149+
143150
assertColumn("select struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:double>>>>), 'k') from tbl",
144151
"struct<city:text>",
145152
ImmutableList.of(path("s", "city")),
146153
ImmutableList.of()
147154
);
148155

149-
assertColumn("select struct_element(map_values(struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:double,y:double>>>>), 'l')[0])[0], 'x') from tbl",
156+
assertColumn("select struct_element(map_values(struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:double>>>>), 'l')[0])[0], 'x') from tbl",
150157
"struct<data:array<map<int,struct<a:int>>>>",
151158
ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
152159
ImmutableList.of()

0 commit comments

Comments
 (0)