Skip to content

[fix](virtual slot) Fix complex type and lambda function #55869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 5 commits into from
Sep 12, 2025
Merged

[fix](virtual slot) Fix complex type and lambda function #55869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
065fae4
FIX ARRAY LAMBDA
zhiqiang-hhhh Sep 10, 2025
ba7aa8f
FS
zhiqiang-hhhh Sep 10, 2025
5c5d158
D
zhiqiang-hhhh Sep 10, 2025
81f85cb
FIX
zhiqiang-hhhh Sep 11, 2025
e8dc35d
FIX FMT
zhiqiang-hhhh Sep 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 15 additions & 51 deletions .../main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
Expand All @@ -41,9 +40,7 @@
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BitmapType;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.CharType;
import org.apache.doris.nereids.types.DataType;
Expand All @@ -55,20 +52,14 @@
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.DoubleType;
import org.apache.doris.nereids.types.FloatType;
import org.apache.doris.nereids.types.HllType;
import org.apache.doris.nereids.types.IPv4Type;
import org.apache.doris.nereids.types.IPv6Type;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.LargeIntType;
import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.QuantileStateType;
import org.apache.doris.nereids.types.SmallIntType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.VariantType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -120,8 +111,9 @@
* 3. CAST Expressions: CAST operations are lightweight and creating virtual columns for them
* may not provide significant benefit while adding complexity.
*
* 4. Lambda-containing Expressions: Expressions with lambda functions have complex evaluation
* contexts that make virtual column optimization problematic.
* 4. Lambda-containing Expressions: Expression trees that contain lambda functions anywhere
* are completely skipped from optimization. Lambda functions have complex evaluation contexts
* that make virtual column optimization problematic.
*/
public class PushDownVirtualColumnsIntoOlapScan implements RewriteRuleFactory {

Expand Down Expand Up @@ -152,17 +144,9 @@ public class PushDownVirtualColumnsIntoOlapScan implements RewriteRuleFactory {
DateTimeV2Type.class,
FloatType.class,
DoubleType.class,
QuantileStateType.class,
BitmapType.class,
CharType.class,
VarcharType.class,
HllType.class,
StringType.class,
VariantType.class,
JsonType.class,
StructType.class,
ArrayType.class,
MapType.class,
DecimalV2Type.class,
DecimalV3Type.class
);
Expand Down Expand Up @@ -282,6 +266,13 @@ private void extractRepeatedSubExpressions(LogicalFilter<LogicalOlapScan> filter
Map<Expression, Integer> expressionCounts = new HashMap<>();

for (Expression expr : allExpressions) {
// Skip expressions that contain lambda functions anywhere in the tree
if (expr.anyMatch(e -> e instanceof Lambda)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping expression containing lambda: {}", expr.toSql());
}
continue;
}
collectSubExpressions(expr, expressionCounts);
}

Expand Down Expand Up @@ -320,22 +311,11 @@ private void extractRepeatedSubExpressions(LogicalFilter<LogicalOlapScan> filter
* Recursively collect all sub-expressions and count their occurrences
*/
private void collectSubExpressions(Expression expr, Map<Expression, Integer> expressionCounts) {
collectSubExpressions(expr, expressionCounts, false);
}

/**
* Recursively collect all sub-expressions and count their occurrences
* @param expr the expression to analyze
* @param expressionCounts map to store expression occurrence counts
* @param insideLambda whether we are currently inside a lambda function
*/
private void collectSubExpressions(Expression expr, Map<Expression, Integer> expressionCounts,
boolean insideLambda) {
// Check if we should skip this expression and how to handle it
SkipResult skipResult = shouldSkipExpression(expr, insideLambda);
SkipResult skipResult = shouldSkipExpression(expr);

if (skipResult.shouldTerminate()) {
// Examples: x (slot), 10 (constant), expressions inside lambda functions
// Examples: x (slot), 10 (constant)
// These expressions are completely skipped - no counting, no recursion
return;
}
Expand All @@ -357,27 +337,17 @@ private void collectSubExpressions(Expression expr, Map<Expression, Integer> exp

// Recursively process children
for (Expression child : expr.children()) {
// Check if we're entering a lambda function
boolean enteringLambda = insideLambda || (expr instanceof Lambda);
collectSubExpressions(child, expressionCounts, enteringLambda);
collectSubExpressions(child, expressionCounts);
}
}

/**
* Determine how to handle an expression during sub-expression collection
* This method consolidates ALL skip logic in one place
* @param expr the expression to check
* @param insideLambda whether we are currently inside a lambda function
* @return SkipResult indicating how to handle this expression
*/
private SkipResult shouldSkipExpression(Expression expr, boolean insideLambda) {
// Skip expressions inside lambda functions - they shouldn't be optimized
if (insideLambda) {
if (expr.containsType(ArrayItemReference.class)) {
return SkipResult.SKIP_NOT_BENEFICIAL;
}
}

private SkipResult shouldSkipExpression(Expression expr) {
// Skip simple slots and literals as they don't benefit from being pushed down
if (expr instanceof Slot || expr.isConstant()) {
return SkipResult.TERMINATE;
Expand All @@ -394,11 +364,6 @@ private SkipResult shouldSkipExpression(Expression expr, boolean insideLambda) {
return SkipResult.SKIP_NOT_BENEFICIAL;
}

// Skip expressions that contain lambda functions anywhere in the tree
if (expr instanceof Lambda) {
return SkipResult.SKIP_NOT_BENEFICIAL;
}

// Skip expressions that can be converted to ColumnPredicate or can use index
// This is the key blacklist logic to avoid reverse optimization
if (canConvertToColumnPredicate(expr) || containsIndexPushdownFunction(expr)) {
Expand Down Expand Up @@ -427,11 +392,10 @@ private enum SkipResult {
// - encode_as_bigint(x), decode_as_varchar(x) - encoding/decoding functions
// - x > 10, x IN (1,2,3) - ColumnPredicate convertible expressions
// - is_ip_address_in_range(ip, '192.168.1.0/24') - index pushdown functions
// - expressions containing lambda functions
SKIP_NOT_BENEFICIAL,

// Stop processing entirely (don't count, don't recurse)
// Examples: x (slot), 10 (constant), expressions inside lambda functions
// Examples: x (slot), 10 (constant)
TERMINATE;

public boolean shouldTerminate() {
Expand Down
Loading