Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,38 @@
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BitmapType;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.CharType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.types.DecimalV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.DoubleType;
import org.apache.doris.nereids.types.FloatType;
import org.apache.doris.nereids.types.HllType;
import org.apache.doris.nereids.types.IPv4Type;
import org.apache.doris.nereids.types.IPv6Type;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.LargeIntType;
import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.QuantileStateType;
import org.apache.doris.nereids.types.SmallIntType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.VariantType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -107,6 +136,38 @@ public class PushDownVirtualColumnsIntoOlapScan implements RewriteRuleFactory {
// Logger for debugging
private static final Logger logger = LogManager.getLogger(PushDownVirtualColumnsIntoOlapScan.class);

// Supported data types for virtual columns based on TabletColumn::get_field_length_by_type
// Using Nereids DataType instead of TPrimitiveType for better type safety
private static final ImmutableSet<Class<? extends DataType>> SUPPORTED_VIRTUAL_COLUMN_TYPES = ImmutableSet.of(
TinyIntType.class,
BooleanType.class,
SmallIntType.class,
IntegerType.class,
BigIntType.class,
LargeIntType.class,
IPv4Type.class,
IPv6Type.class,
DateType.class,
DateV2Type.class,
DateTimeType.class,
DateTimeV2Type.class,
FloatType.class,
DoubleType.class,
QuantileStateType.class,
BitmapType.class,
CharType.class,
VarcharType.class,
HllType.class,
StringType.class,
VariantType.class,
JsonType.class,
StructType.class,
ArrayType.class,
MapType.class,
DecimalV2Type.class,
DecimalV3Type.class
);

@Override
public List<Rule> buildRules() {
return ImmutableList.of(
Expand Down Expand Up @@ -205,6 +266,7 @@ private void extractRepeatedSubExpressions(LogicalFilter<LogicalOlapScan> filter
expressionCounts.entrySet().stream()
.filter(entry -> entry.getValue() >= MIN_OCCURRENCE_COUNT)
.filter(entry -> !replaceMap.containsKey(entry.getKey()))
.filter(entry -> isSupportedVirtualColumnType(entry.getKey())) // Check virtual column type support
.sorted((e1, e2) -> {
// Sort by benefit: (occurrence_count - 1) * expression_complexity
int benefit1 = (e1.getValue() - 1) * getExpressionComplexity(e1.getKey());
Expand All @@ -217,6 +279,11 @@ private void extractRepeatedSubExpressions(LogicalFilter<LogicalOlapScan> filter
Alias alias = new Alias(expr);
replaceMap.put(expr, alias.toSlot());
virtualColumnsBuilder.add(alias);

if (LOG.isDebugEnabled()) {
LOG.debug("Created virtual column for expression: {} with type: {}",
expr.toSql(), expr.getDataType().simpleString());
}
});

// Logging for debugging
Expand Down Expand Up @@ -438,6 +505,32 @@ private boolean isIndexPushdownFunction(Expression expr) {
}
}

/**
* Check if an expression's return type is supported for virtual columns
* Based on TabletColumn::get_field_length_by_type implementation
*/
private boolean isSupportedVirtualColumnType(Expression expr) {
try {
DataType dataType = expr.getDataType();
Class<? extends DataType> typeClass = dataType.getClass();

boolean isSupported = SUPPORTED_VIRTUAL_COLUMN_TYPES.contains(typeClass);

if (LOG.isDebugEnabled()) {
LOG.debug("Expression {} has type {} (class: {}), supported: {}",
expr.toSql(), dataType.simpleString(), typeClass.getSimpleName(), isSupported);
}

return isSupported;
} catch (Exception e) {
// If we can't determine the type, it's safer to not create virtual column
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to determine type for expression {}: {}", expr.toSql(), e.getMessage());
}
return false;
}
}

/**
* Get function name from expression if it's a function call
*/
Expand Down
Loading
Loading