From 065fae43d54aef082cc0a5170a1172b548abd379 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Wed, 10 Sep 2025 17:18:57 +0800 Subject: [PATCH 1/5] FIX ARRAY LAMBDA --- .../PushDownVirtualColumnsIntoOlapScan.java | 52 +--- ...ushDownVirtualColumnsIntoOlapScanTest.java | 282 ++++++++++++++++-- .../fix_array_type_and_lambda_func.out | 16 + .../fix_array_type_and_lambda_func.groovy | 47 +++ 4 files changed, 342 insertions(+), 55 deletions(-) create mode 100644 regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out create mode 100644 regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java index bb6729a9d67fc6..60fdcd3794269f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.ArrayItemReference; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; @@ -41,7 +40,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; import org.apache.doris.nereids.types.BooleanType; @@ -120,8 +118,9 @@ * 3. CAST Expressions: CAST operations are lightweight and creating virtual columns for them * may not provide significant benefit while adding complexity. * - * 4. Lambda-containing Expressions: Expressions with lambda functions have complex evaluation - * contexts that make virtual column optimization problematic. + * 4. Lambda-containing Expressions: Expression trees that contain lambda functions anywhere + * are completely skipped from optimization. Lambda functions have complex evaluation contexts + * that make virtual column optimization problematic. */ public class PushDownVirtualColumnsIntoOlapScan implements RewriteRuleFactory { @@ -161,7 +160,6 @@ public class PushDownVirtualColumnsIntoOlapScan implements RewriteRuleFactory { VariantType.class, JsonType.class, StructType.class, - ArrayType.class, MapType.class, DecimalV2Type.class, DecimalV3Type.class @@ -282,6 +280,13 @@ private void extractRepeatedSubExpressions(LogicalFilter filter Map expressionCounts = new HashMap<>(); for (Expression expr : allExpressions) { + // Skip expressions that contain lambda functions anywhere in the tree + if (expr.anyMatch(e -> e instanceof Lambda)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping expression containing lambda: {}", expr.toSql()); + } + continue; + } collectSubExpressions(expr, expressionCounts); } @@ -320,22 +325,11 @@ private void extractRepeatedSubExpressions(LogicalFilter filter * Recursively collect all sub-expressions and count their occurrences */ private void collectSubExpressions(Expression expr, Map expressionCounts) { - collectSubExpressions(expr, expressionCounts, false); - } - - /** - * Recursively collect all sub-expressions and count their occurrences - * @param expr the expression to analyze - * @param expressionCounts map to store expression occurrence counts - * @param insideLambda whether we are currently inside a lambda function - */ - private void collectSubExpressions(Expression expr, Map expressionCounts, - boolean insideLambda) { // Check if we should skip this expression and how to handle it - SkipResult skipResult = shouldSkipExpression(expr, insideLambda); + SkipResult skipResult = shouldSkipExpression(expr); if (skipResult.shouldTerminate()) { - // Examples: x (slot), 10 (constant), expressions inside lambda functions + // Examples: x (slot), 10 (constant) // These expressions are completely skipped - no counting, no recursion return; } @@ -357,9 +351,7 @@ private void collectSubExpressions(Expression expr, Map exp // Recursively process children for (Expression child : expr.children()) { - // Check if we're entering a lambda function - boolean enteringLambda = insideLambda || (expr instanceof Lambda); - collectSubExpressions(child, expressionCounts, enteringLambda); + collectSubExpressions(child, expressionCounts); } } @@ -367,17 +359,9 @@ private void collectSubExpressions(Expression expr, Map exp * Determine how to handle an expression during sub-expression collection * This method consolidates ALL skip logic in one place * @param expr the expression to check - * @param insideLambda whether we are currently inside a lambda function * @return SkipResult indicating how to handle this expression */ - private SkipResult shouldSkipExpression(Expression expr, boolean insideLambda) { - // Skip expressions inside lambda functions - they shouldn't be optimized - if (insideLambda) { - if (expr.containsType(ArrayItemReference.class)) { - return SkipResult.SKIP_NOT_BENEFICIAL; - } - } - + private SkipResult shouldSkipExpression(Expression expr) { // Skip simple slots and literals as they don't benefit from being pushed down if (expr instanceof Slot || expr.isConstant()) { return SkipResult.TERMINATE; @@ -394,11 +378,6 @@ private SkipResult shouldSkipExpression(Expression expr, boolean insideLambda) { return SkipResult.SKIP_NOT_BENEFICIAL; } - // Skip expressions that contain lambda functions anywhere in the tree - if (expr instanceof Lambda) { - return SkipResult.SKIP_NOT_BENEFICIAL; - } - // Skip expressions that can be converted to ColumnPredicate or can use index // This is the key blacklist logic to avoid reverse optimization if (canConvertToColumnPredicate(expr) || containsIndexPushdownFunction(expr)) { @@ -427,11 +406,10 @@ private enum SkipResult { // - encode_as_bigint(x), decode_as_varchar(x) - encoding/decoding functions // - x > 10, x IN (1,2,3) - ColumnPredicate convertible expressions // - is_ip_address_in_range(ip, '192.168.1.0/24') - index pushdown functions - // - expressions containing lambda functions SKIP_NOT_BENEFICIAL, // Stop processing entirely (don't count, don't recurse) - // Examples: x (slot), 10 (constant), expressions inside lambda functions + // Examples: x (slot), 10 (constant) TERMINATE; public boolean shouldTerminate() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScanTest.java index 66b78c90216846..1603252154d564 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScanTest.java @@ -71,6 +71,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Test for PushDownVirtualColumnsIntoOlapScan rule. @@ -309,8 +310,8 @@ public void testSkipCastExpressions() { @Test public void testSkipLambdaExpressions() { - // Test that expressions inside lambda functions are not optimized - // This is a simplified test since creating actual lambda expressions is complex + // Test that expressions containing lambda functions are completely skipped from optimization + // With the new logic, any expression tree containing lambda should not be optimized ConnectContext connectContext = MemoTestUtils.createConnectContext(); @@ -326,11 +327,11 @@ public void testSkipLambdaExpressions() { Add lambdaAdd = new Add(refA.toSlot(), xyAdd); Lambda lambda = new Lambda(ImmutableList.of("a"), lambdaAdd, ImmutableList.of(refA)); - // Create two expression contain lambda + // Create two expressions containing lambda ArrayFilter arrayFilter = new ArrayFilter(lambda); ArrayMap arrayMap = new ArrayMap(lambda); - // Create filter + // Create filter with expressions containing lambda LogicalFilter filter = new LogicalFilter<>( ImmutableSet.of(new EqualTo(arrayFilter, arrayMap)), scan); @@ -341,8 +342,7 @@ public void testSkipLambdaExpressions() { // Test rule creation Assertions.assertEquals(2, rules.size()); - // This test verifies the rule structure but actual lambda testing would require - // more complex expression trees with lambda functions + // This test verifies that expressions containing lambda are completely skipped boolean hasFilterScanRule = false; for (Rule r : rules) { if (r.getPattern().matchPlanTree(filter)) { @@ -352,20 +352,10 @@ public void testSkipLambdaExpressions() { } Assertions.assertTrue(hasFilterScanRule, "Should have rule that matches filter->scan pattern"); + // With the new logic, expressions containing lambda should NOT create any virtual columns PlanChecker.from(connectContext, filter) .applyTopDown(rules) - .applyCustom(new ColumnPruning()) - .matches(logicalOlapScan() - .when(o -> o.getVirtualColumns().size() == 2) - .when(o -> { - for (NamedExpression virtualColumn : o.getVirtualColumns()) { - Expression c = virtualColumn.child(0); - if (!(c instanceof ArrayMap) && !c.equals(arr)) { - return false; - } - } - return true; - })); + .matches(logicalOlapScan().when(o -> o.getVirtualColumns().isEmpty())); } @Test @@ -912,4 +902,260 @@ void testMultipleTimesUniqueFunctions() { ); Assertions.assertEquals(expectConjuncts, resFilter.getConjuncts()); } + + @Test + public void testCompleteSkipOfLambdaContainingExpressions() { + // Test that any expression tree containing lambda anywhere is completely skipped + + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference x = (SlotReference) scan.getOutput().get(0); + SlotReference y = (SlotReference) scan.getOutput().get(1); + + // Create regular repeated expressions (should be optimized) + Add regularAdd1 = new Add(x, y); + Add regularAdd2 = new Add(x, y); + + // Create lambda expression + Array arr = new Array(y); + ArrayItemReference refA = new ArrayItemReference("a", arr); + Add lambdaAdd = new Add(refA.toSlot(), x); + Lambda lambda = new Lambda(ImmutableList.of("a"), lambdaAdd, ImmutableList.of(refA)); + + // Create expressions that contain lambda at different levels + ArrayFilter arrayFilter1 = new ArrayFilter(lambda); // direct lambda usage + ArrayFilter arrayFilter2 = new ArrayFilter(lambda); // repeated lambda usage + Add nestedWithLambda = new Add(arrayFilter1, x); // lambda in nested expression + + // Mix regular repeated expressions with lambda-containing expressions + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of( + new GreaterThan(regularAdd1, new IntegerLiteral(10)), // should be optimized + new LessThan(regularAdd2, new IntegerLiteral(100)), // should be optimized + new EqualTo(arrayFilter1, arrayFilter2), // should NOT be optimized (contains lambda) + new GreaterThan(nestedWithLambda, new IntegerLiteral(5)) // should NOT be optimized (contains lambda) + ), scan); + + // Apply the rule + PushDownVirtualColumnsIntoOlapScan rule = new PushDownVirtualColumnsIntoOlapScan(); + List rules = rule.buildRules(); + + Plan result = PlanChecker.from(connectContext, filter) + .applyTopDown(rules) + .getPlan(); + + // Verify that only regular expressions without lambda are optimized + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject project = (LogicalProject) result; + Assertions.assertInstanceOf(LogicalFilter.class, project.child()); + LogicalFilter resFilter = (LogicalFilter) project.child(); + Assertions.assertInstanceOf(LogicalOlapScan.class, resFilter.child()); + LogicalOlapScan resScan = (LogicalOlapScan) resFilter.child(); + + // Should have exactly 1 virtual column for the regular Add expression + Assertions.assertEquals(1, resScan.getVirtualColumns().size()); + Alias alias = (Alias) resScan.getVirtualColumns().get(0); + Assertions.assertEquals(regularAdd1, alias.child()); // The regular Add(x, y) expression + + // Verify that lambda-containing expressions are NOT replaced + boolean foundLambdaExpressions = resFilter.getConjuncts().stream() + .anyMatch(expr -> expr.anyMatch(e -> e instanceof ArrayFilter || e instanceof Lambda)); + Assertions.assertTrue(foundLambdaExpressions, + "Lambda-containing expressions should remain unchanged in filter"); + } + + @Test + public void testLambdaInProjectExpressions() { + // Test that lambda-containing expressions in project are also skipped + + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference x = (SlotReference) scan.getOutput().get(0); + SlotReference y = (SlotReference) scan.getOutput().get(1); + + // Create regular repeated expressions + Add regularAdd1 = new Add(x, y); + Add regularAdd2 = new Add(x, y); + + // Create lambda expression used in project + Array arr = new Array(y); + ArrayItemReference refA = new ArrayItemReference("a", arr); + Add lambdaAdd = new Add(refA.toSlot(), x); + Lambda lambda = new Lambda(ImmutableList.of("a"), lambdaAdd, ImmutableList.of(refA)); + ArrayMap arrayMap1 = new ArrayMap(lambda); + ArrayMap arrayMap2 = new ArrayMap(lambda); // repeated lambda expression + + // Create filter with regular expressions + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of( + new GreaterThan(regularAdd1, new IntegerLiteral(10)), + new LessThan(regularAdd2, new IntegerLiteral(100)) + ), scan); + + // Create project with lambda-containing expressions + List projects = ImmutableList.of( + new Alias(arrayMap1, "lambda_result1"), + new Alias(arrayMap2, "lambda_result2"), + new Alias(x, "x_col") + ); + LogicalProject> project = + new LogicalProject<>(projects, filter); + + // Apply the rule + PushDownVirtualColumnsIntoOlapScan rule = new PushDownVirtualColumnsIntoOlapScan(); + List rules = rule.buildRules(); + + Plan result = PlanChecker.from(connectContext, project) + .applyTopDown(rules) + .getPlan(); + + // Verify optimization results + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject resProject = (LogicalProject) result; + Assertions.assertInstanceOf(LogicalFilter.class, resProject.child()); + LogicalFilter resFilter = (LogicalFilter) resProject.child(); + Assertions.assertInstanceOf(LogicalOlapScan.class, resFilter.child()); + LogicalOlapScan resScan = (LogicalOlapScan) resFilter.child(); + + // Should have exactly 1 virtual column for the regular Add expression + Assertions.assertEquals(1, resScan.getVirtualColumns().size()); + Alias alias = (Alias) resScan.getVirtualColumns().get(0); + Assertions.assertEquals(regularAdd1, alias.child()); + + // Verify that lambda expressions in project are NOT replaced + boolean foundLambdaInProject = resProject.getProjects().stream() + .anyMatch(expr -> expr.anyMatch(e -> e instanceof ArrayMap || e instanceof Lambda)); + Assertions.assertTrue(foundLambdaInProject, + "Lambda-containing expressions should remain unchanged in project"); + } + + @Test + public void testNestedLambdaExpressions() { + // Test deeply nested expressions containing lambda are completely skipped + + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference x = (SlotReference) scan.getOutput().get(0); + SlotReference y = (SlotReference) scan.getOutput().get(1); + + // Create regular repeated expressions + Add regularAdd1 = new Add(x, y); + Add regularAdd2 = new Add(x, y); + + // Create deeply nested expression with lambda + Array arr = new Array(y); + ArrayItemReference refA = new ArrayItemReference("a", arr); + Lambda lambda = new Lambda(ImmutableList.of("a"), new Add(refA.toSlot(), x), ImmutableList.of(refA)); + ArrayFilter arrayFilter = new ArrayFilter(lambda); + + // Create complex nested expression containing lambda + Multiply complexWithLambda1 = new Multiply( + new Add(arrayFilter, x), + new IntegerLiteral(2) + ); + Multiply complexWithLambda2 = new Multiply( + new Add(arrayFilter, x), + new IntegerLiteral(2) + ); // repeated complex expression with lambda + + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of( + new GreaterThan(regularAdd1, new IntegerLiteral(10)), // should be optimized + new LessThan(regularAdd2, new IntegerLiteral(100)), // should be optimized + new GreaterThan(complexWithLambda1, new IntegerLiteral(5)), // should NOT be optimized + new LessThan(complexWithLambda2, new IntegerLiteral(50)) // should NOT be optimized + ), scan); + + // Apply the rule + PushDownVirtualColumnsIntoOlapScan rule = new PushDownVirtualColumnsIntoOlapScan(); + List rules = rule.buildRules(); + + Plan result = PlanChecker.from(connectContext, filter) + .applyTopDown(rules) + .getPlan(); + + // Verify results + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject project = (LogicalProject) result; + Assertions.assertInstanceOf(LogicalFilter.class, project.child()); + LogicalFilter resFilter = (LogicalFilter) project.child(); + Assertions.assertInstanceOf(LogicalOlapScan.class, resFilter.child()); + LogicalOlapScan resScan = (LogicalOlapScan) resFilter.child(); + + // Should have exactly 1 virtual column for the regular Add expression only + Assertions.assertEquals(1, resScan.getVirtualColumns().size()); + Alias alias = (Alias) resScan.getVirtualColumns().get(0); + Assertions.assertEquals(regularAdd1, alias.child()); + + // Verify that nested lambda expressions remain unchanged + boolean foundComplexLambdaExpressions = resFilter.getConjuncts().stream() + .anyMatch(expr -> expr.anyMatch(e -> e instanceof Multiply + && e.anyMatch(inner -> inner instanceof ArrayFilter))); + Assertions.assertTrue(foundComplexLambdaExpressions, + "Complex expressions containing lambda should remain unchanged"); + } + + @Test + public void testMixedExpressionsWithAndWithoutLambda() { + // Test comprehensive scenario mixing lambda and non-lambda expressions + + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference x = (SlotReference) scan.getOutput().get(0); + SlotReference y = (SlotReference) scan.getOutput().get(1); + + // Regular repeated expressions (should be optimized) + Add regularAdd1 = new Add(x, y); + Add regularAdd2 = new Add(x, y); + Multiply regularMult1 = new Multiply(x, new IntegerLiteral(3)); + Multiply regularMult2 = new Multiply(x, new IntegerLiteral(3)); + + // Lambda expressions (should NOT be optimized) + Array arr = new Array(y); + ArrayItemReference refA = new ArrayItemReference("a", arr); + Lambda lambda = new Lambda(ImmutableList.of("a"), new Add(refA.toSlot(), x), ImmutableList.of(refA)); + ArrayFilter lambdaExpr1 = new ArrayFilter(lambda); + ArrayFilter lambdaExpr2 = new ArrayFilter(lambda); // repeated but contains lambda + + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of( + new GreaterThan(regularAdd1, new IntegerLiteral(10)), // optimizable + new LessThan(regularAdd2, new IntegerLiteral(100)), // optimizable + new GreaterThan(regularMult1, new IntegerLiteral(5)), // optimizable + new LessThan(regularMult2, new IntegerLiteral(50)), // optimizable + new EqualTo(lambdaExpr1, lambdaExpr2) // NOT optimizable (lambda) + ), scan); + + // Apply the rule + PushDownVirtualColumnsIntoOlapScan rule = new PushDownVirtualColumnsIntoOlapScan(); + List rules = rule.buildRules(); + + Plan result = PlanChecker.from(connectContext, filter) + .applyTopDown(rules) + .getPlan(); + + // Verify results + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject project = (LogicalProject) result; + Assertions.assertInstanceOf(LogicalFilter.class, project.child()); + LogicalFilter resFilter = (LogicalFilter) project.child(); + Assertions.assertInstanceOf(LogicalOlapScan.class, resFilter.child()); + LogicalOlapScan resScan = (LogicalOlapScan) resFilter.child(); + + // Should have exactly 2 virtual columns for the two regular repeated expressions + Assertions.assertEquals(2, resScan.getVirtualColumns().size()); + + // Check that virtual columns contain the expected regular expressions + Set virtualColumnExprs = resScan.getVirtualColumns().stream() + .map(vc -> ((Alias) vc).child()) + .collect(Collectors.toSet()); + + Assertions.assertTrue(virtualColumnExprs.contains(regularAdd1) || virtualColumnExprs.contains(regularAdd2)); + Assertions.assertTrue(virtualColumnExprs.contains(regularMult1) || virtualColumnExprs.contains(regularMult2)); + + // Verify that lambda expressions remain in filter + boolean hasLambdaInFilter = resFilter.getConjuncts().stream() + .anyMatch(expr -> expr.anyMatch(e -> e instanceof ArrayFilter)); + Assertions.assertTrue(hasLambdaInFilter, "Lambda expressions should remain in filter unchanged"); + } } diff --git a/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out b/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out new file mode 100644 index 00000000000000..58090aff9e713b --- /dev/null +++ b/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !0 -- +[] [] + +-- !1 -- +[] [] +[] [] +[] [] +[] [] +[2147483647] [2147483647] +[5] [5] +[200, 300] [200, 300] +[] [] +[] [] +[] [] + diff --git a/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy b/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy new file mode 100644 index 00000000000000..d16d4ed2a30a64 --- /dev/null +++ b/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("fix_array_type_and_lambda_func") { + sql """DROP TABLE IF EXISTS fix_array_type_and_lambda_func""" + sql """ + CREATE TABLE + fix_array_type_and_lambda_func (id INT, arr_col ARRAY, int_col INT, str_col VARCHAR(50)) + DISTRIBUTED BY HASH(id) BUCKETS 4 PROPERTIES ("replication_num" = "1"); + """ + sql """ + INSERT INTO fix_array_type_and_lambda_func (id, arr_col, int_col, str_col) + VALUES (1, [1,2,3], 5, 'normal'), (2, [], 0, 'empty'), (3, [2147483647, -2147483648], 100, 'boundary'), (4, NULL, NULL, 'null'), + (5, [1,2,3], 3, 'special$char'); + """ + // return type of array_with_constant is array. + // do not optimizate + qt_0 """ + SELECT array_with_constant(int_col, NULL) AS arr1, array_with_constant(int_col, NULL) AS arr2 FROM fix_array_type_and_lambda_func WHERE id = 4 + """ + + sql """ + CREATE TABLE test_array_filter (id INT, arr_col ARRAY, str_col VARCHAR(20), int_col INT) DISTRIBUTED BY HASH(id) BUCKETS 4 PROPERTIES ("replication_num" = "1"); + """ + + sql """ + INSERT INTO test_array_filter VALUES (1, [1,2,3,4,5], 'test1', 10), (2, [10,20,30], 'test2', 25), (3, [-5,0,5], 'test3', -3), (4, [], 'empty', 0), (5, [100,200,300], 'large', 150), (6, [1,1,2,3,5], 'fibonacci', 8), (7, [NULL,1,2], 'with_null', 1), (8, [2147483647,-2147483648], 'boundary', 0), (9, [1,2,3], 'normal', 5), (10, [0,0,0], 'zeros', 0); + """ + + qt_1 """ + SELECT array_filter(x -> x > int_col + 5, arr_col) AS expr1, array_filter(x -> x > int_col + 5, arr_col) AS expr2 FROM test_array_filter WHERE array_filter(x -> x > int_col + 5, arr_col) IS NOT NULL; + """ +} \ No newline at end of file From ba7aa8f6d3ddd1057aec309b963b14c9643fd574 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Wed, 10 Sep 2025 17:30:20 +0800 Subject: [PATCH 2/5] FS --- .../fix_array_type_and_lambda_func.out | 6 +++--- .../fix_array_type_and_lambda_func.groovy | 13 +++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out b/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out index 58090aff9e713b..8005192abb7938 100644 --- a/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out +++ b/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out @@ -5,12 +5,12 @@ -- !1 -- [] [] [] [] -[] [] -[] [] -[2147483647] [2147483647] [5] [5] +[] [] [200, 300] [200, 300] [] [] [] [] +[2147483647] [2147483647] +[] [] [] [] diff --git a/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy b/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy index d16d4ed2a30a64..c905ae1d590ecf 100644 --- a/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy +++ b/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy @@ -32,7 +32,9 @@ suite("fix_array_type_and_lambda_func") { qt_0 """ SELECT array_with_constant(int_col, NULL) AS arr1, array_with_constant(int_col, NULL) AS arr2 FROM fix_array_type_and_lambda_func WHERE id = 4 """ - + sql """ + drop table if exists test_array_filter; + """ sql """ CREATE TABLE test_array_filter (id INT, arr_col ARRAY, str_col VARCHAR(20), int_col INT) DISTRIBUTED BY HASH(id) BUCKETS 4 PROPERTIES ("replication_num" = "1"); """ @@ -42,6 +44,13 @@ suite("fix_array_type_and_lambda_func") { """ qt_1 """ - SELECT array_filter(x -> x > int_col + 5, arr_col) AS expr1, array_filter(x -> x > int_col + 5, arr_col) AS expr2 FROM test_array_filter WHERE array_filter(x -> x > int_col + 5, arr_col) IS NOT NULL; + WITH filtered_data AS ( + SELECT id, + array_filter(x -> x > int_col + 5, arr_col) AS expr1, + array_filter(x -> x > int_col + 5, arr_col) AS expr2 + FROM test_array_filter + WHERE array_filter(x -> x > int_col + 5, arr_col) IS NOT NULL + ) + SELECT expr1, expr2 FROM filtered_data ORDER BY id; """ } \ No newline at end of file From 5c5d1585f7edc46006be8975b834edd4d53503ef Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Wed, 10 Sep 2025 17:35:50 +0800 Subject: [PATCH 3/5] D --- .../fix_array_type_and_lambda_func.out | 19 +++++++++---------- .../fix_array_type_and_lambda_func.groovy | 7 ++++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out b/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out index 8005192abb7938..36f5be578f2b68 100644 --- a/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out +++ b/regression-test/data/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.out @@ -3,14 +3,13 @@ [] [] -- !1 -- -[] [] -[] [] -[5] [5] -[] [] -[200, 300] [200, 300] -[] [] -[] [] -[2147483647] [2147483647] -[] [] -[] [] +[] [] 2 +[5] [5] 3 +[] [] 4 +[200, 300] [200, 300] 5 +[] [] 6 +[] [] 7 +[2147483647] [2147483647] 8 +[] [] 9 +[] [] 10 diff --git a/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy b/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy index c905ae1d590ecf..9d8605de2ec840 100644 --- a/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy +++ b/regression-test/suites/query_p0/virtual_slot_ref/fix_array_type_and_lambda_func.groovy @@ -45,12 +45,13 @@ suite("fix_array_type_and_lambda_func") { qt_1 """ WITH filtered_data AS ( - SELECT id, + SELECT id, + round(id), array_filter(x -> x > int_col + 5, arr_col) AS expr1, array_filter(x -> x > int_col + 5, arr_col) AS expr2 FROM test_array_filter - WHERE array_filter(x -> x > int_col + 5, arr_col) IS NOT NULL + WHERE array_filter(x -> x > int_col + 5, arr_col) IS NOT NULL AND round(id) > 1 ) - SELECT expr1, expr2 FROM filtered_data ORDER BY id; + SELECT expr1, expr2, round(id) FROM filtered_data ORDER BY id; """ } \ No newline at end of file From 81f85cb64d41d2f14333ed444404574be6367fd4 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Thu, 11 Sep 2025 10:58:12 +0800 Subject: [PATCH 4/5] FIX --- .../rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java index 60fdcd3794269f..61f6a2bde580d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java @@ -151,16 +151,9 @@ public class PushDownVirtualColumnsIntoOlapScan implements RewriteRuleFactory { DateTimeV2Type.class, FloatType.class, DoubleType.class, - QuantileStateType.class, - BitmapType.class, CharType.class, VarcharType.class, - HllType.class, StringType.class, - VariantType.class, - JsonType.class, - StructType.class, - MapType.class, DecimalV2Type.class, DecimalV3Type.class ); From e8dc35d28a6ce5a986d2b2eec3d2a168f3e573c7 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Thu, 11 Sep 2025 11:45:04 +0800 Subject: [PATCH 5/5] FIX FMT --- .../rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java index 61f6a2bde580d2..0f17f7910fad89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirtualColumnsIntoOlapScan.java @@ -41,7 +41,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.types.BigIntType; -import org.apache.doris.nereids.types.BitmapType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; @@ -53,20 +52,14 @@ import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.FloatType; -import org.apache.doris.nereids.types.HllType; import org.apache.doris.nereids.types.IPv4Type; import org.apache.doris.nereids.types.IPv6Type; import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.types.JsonType; import org.apache.doris.nereids.types.LargeIntType; -import org.apache.doris.nereids.types.MapType; -import org.apache.doris.nereids.types.QuantileStateType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; -import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList;