From be1b1a6c27e38f35530fb01295cb89e84a76e22b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 30 Oct 2025 15:44:44 +0800 Subject: [PATCH 01/58] add code --- .../java/org/apache/doris/alter/Alter.java | 12 +-- .../java/org/apache/doris/catalog/Column.java | 30 +++++-- .../java/org/apache/doris/catalog/Env.java | 5 +- .../java/org/apache/doris/catalog/View.java | 18 +++- .../doris/nereids/StatementContext.java | 10 +-- .../doris/nereids/minidump/MinidumpUtils.java | 4 +- .../nereids/parser/LogicalPlanBuilder.java | 2 +- .../nereids/rules/analysis/BindRelation.java | 12 +-- .../nereids/rules/analysis/BindSink.java | 27 +++--- .../rules/analysis/CollectRelation.java | 11 ++- .../rules/SimplifyComparisonPredicate.java | 6 +- .../doris/nereids/trees/expressions/Add.java | 6 +- .../trees/expressions/BinaryArithmetic.java | 31 ++++++- .../nereids/trees/expressions/Divide.java | 10 ++- .../doris/nereids/trees/expressions/Mod.java | 6 +- .../nereids/trees/expressions/Multiply.java | 6 +- .../nereids/trees/expressions/Params.java | 50 +++++++++++ .../nereids/trees/expressions/Subtract.java | 6 +- .../DateTimeExtractAndTransform.java | 4 +- .../executable/NumericArithmetic.java | 24 ++++-- .../expressions/literal/DecimalV3Literal.java | 16 +++- .../trees/expressions/literal/Literal.java | 2 +- .../plans/commands/info/ColumnDefinition.java | 10 ++- .../doris/nereids/types/DecimalV2Type.java | 2 +- .../doris/nereids/types/DecimalV3Type.java | 57 +++++-------- .../apache/doris/persist/AlterViewInfo.java | 20 +++-- .../doris/qe/AutoCloseSessionVariable.java | 62 ++++++++++++++ .../apache/doris/qe/ConnectContextUtil.java | 8 ++ .../org/apache/doris/qe/SessionVariable.java | 71 +++++++++++----- .../java/org/apache/doris/qe/VariableMgr.java | 83 ++++++++++--------- .../apache/doris/catalog/CreateViewTest.java | 5 +- .../apache/doris/qe/OlapQueryCacheTest.java | 9 +- 32 files changed, 431 insertions(+), 194 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Params.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 10ea0c5e55f735..142ff176657d4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -97,6 +97,7 @@ import org.apache.doris.persist.ReplaceTableOperationLog; import org.apache.doris.policy.StoragePolicy; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.resource.Tag; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TOdbcTableType; @@ -836,11 +837,12 @@ public void processAlterView(AlterViewCommand command, ConnectContext ctx) throw String tableName = tableNameInfo.getTbl(); View view = (View) db.getTableOrMetaException(tableName, TableType.VIEW); - modifyViewDef(db, view, alterViewInfo.getInlineViewDef(), ctx.getSessionVariable().getSqlMode(), + modifyViewDef(db, view, alterViewInfo.getInlineViewDef(), + ConnectContextUtil.getAffectQueryResultSessionVariables(ConnectContext.get()), alterViewInfo.getColumns(), alterViewInfo.getComment()); } - private void modifyViewDef(Database db, View view, String inlineViewDef, long sqlMode, + private void modifyViewDef(Database db, View view, String inlineViewDef, Map variables, List newFullSchema, String comment) throws DdlException { db.writeLockOrDdlException(); try { @@ -851,14 +853,14 @@ private void modifyViewDef(Database db, View view, String inlineViewDef, long sq } // when do alter view modify comment, inlineViewDef and newFullSchema will be empty. if (!Strings.isNullOrEmpty(inlineViewDef)) { - view.setInlineViewDefWithSqlMode(inlineViewDef, sqlMode); + view.setInlineViewDefWithSessionVariables(inlineViewDef, variables); view.setNewFullSchema(newFullSchema); } String viewName = view.getName(); db.unregisterTable(viewName); db.registerTable(view); AlterViewInfo alterViewInfo = new AlterViewInfo(db.getId(), view.getId(), - inlineViewDef, newFullSchema, sqlMode, comment); + inlineViewDef, newFullSchema, variables, comment); Env.getCurrentEnv().getMtmvService().alterView(new BaseTableInfo(view)); Env.getCurrentEnv().getEditLog().logModifyViewDef(alterViewInfo); LOG.info("modify view[{}] definition to {}", viewName, inlineViewDef); @@ -887,7 +889,7 @@ public void replayModifyViewDef(AlterViewInfo alterViewInfo) throws MetaNotFound if (comment != null) { view.setComment(comment); } else { - view.setInlineViewDefWithSqlMode(inlineViewDef, alterViewInfo.getSqlMode()); + view.setInlineViewDefWithSessionVariables(inlineViewDef, alterViewInfo.getSessionVariables()); view.setNewFullSchema(newFullSchema); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 05f4b5da3bc890..1a836632219a27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -159,6 +159,9 @@ public class Column implements GsonPostProcessable { @SerializedName(value = "ei") private String extraInfo; + @SerializedName(value = "sv") + private Map sessionVariables; + public Column() { this.name = ""; this.type = Type.NULL; @@ -168,6 +171,7 @@ public Column() { this.defineExpr = null; this.children = null; this.uniqueId = -1; + this.sessionVariables = null; } public Column(String name, PrimitiveType dataType) { @@ -203,40 +207,41 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType String defaultValue, String comment) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, true, null, COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue, false, null, null, - Sets.newHashSet()); + Sets.newHashSet(), null); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String comment, boolean visible, int colUniqueId) { this(name, type, isKey, aggregateType, isAllowNull, -1, null, comment, visible, null, colUniqueId, null, - false, null, null, Sets.newHashSet()); + false, null, null, Sets.newHashSet(), null); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment, boolean visible, int colUniqueId) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, null, colUniqueId, null, - false, null, null, Sets.newHashSet()); + false, null, null, Sets.newHashSet(), null); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, defaultValueExprDef, - colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet()); + colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet(), null); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, - defaultValueExprDef, colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet()); + defaultValueExprDef, colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet(), null); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, boolean hasOnUpdateDefaultValue, DefaultValueExprDef onUpdateDefaultValueExprDef, - GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis, + Map sessionVariables) { this.name = name; if (this.name == null) { this.name = ""; @@ -267,6 +272,7 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType if (CollectionUtils.isNotEmpty(generatedColumnsThatReferToThis)) { this.generatedColumnsThatReferToThis = new HashSet<>(generatedColumnsThatReferToThis); } + this.sessionVariables = sessionVariables; if (type.isAggStateType()) { AggStateType aggState = (AggStateType) type; @@ -285,11 +291,12 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, boolean hasOnUpdateDefaultValue, DefaultValueExprDef onUpdateDefaultValueExprDef, int clusterKeyId, - GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis, + Map sessionVariables) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, defaultValueExprDef, colUniqueId, realDefaultValue, hasOnUpdateDefaultValue, onUpdateDefaultValueExprDef, generatedColumnInfo, - generatedColumnsThatReferToThis); + generatedColumnsThatReferToThis, sessionVariables); this.clusterKeyId = clusterKeyId; } @@ -299,7 +306,7 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, defaultValueExprDef, colUniqueId, realDefaultValue, false, null, generatedColumnInfo, - generatedColumnsThatReferToThis); + generatedColumnsThatReferToThis, null); this.clusterKeyId = clusterKeyId; } @@ -325,6 +332,7 @@ public Column(Column column) { this.onUpdateDefaultValueExprDef = column.onUpdateDefaultValueExprDef; this.clusterKeyId = column.getClusterKeyId(); this.generatedColumnInfo = column.generatedColumnInfo; + this.sessionVariables = column.sessionVariables; } public void createChildrenColumn(Type type, Column column) { @@ -1311,4 +1319,8 @@ public TPatternType getFieldPatternType() { public String getExtraInfo() { return extraInfo; } + + public Map getSessionVariables() { + return sessionVariables; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 7d8e59b708bede..a839cc71e71a61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -231,6 +231,7 @@ import org.apache.doris.policy.PolicyMgr; import org.apache.doris.qe.AuditEventProcessor; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.qe.FEOpExecutor; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.JournalObservable; @@ -6316,8 +6317,8 @@ public void createView(CreateViewCommand command) throws DdlException { long tableId = Env.getCurrentEnv().getNextId(); View newView = new View(tableId, tableName, columns); newView.setComment(createViewInfo.getComment()); - newView.setInlineViewDefWithSqlMode(createViewInfo.getInlineViewDef(), - ConnectContext.get().getSessionVariable().getSqlMode()); + newView.setInlineViewDefWithSessionVariables(createViewInfo.getInlineViewDef(), + ConnectContextUtil.getAffectQueryResultSessionVariables(ConnectContext.get())); if (!((Database) db).createTableWithLock(newView, false, createViewInfo.isIfNotExists()).first) { throw new DdlException("Failed to create view[" + tableName + "]."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java index 84ce2d0a9a8294..180969775fbec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java @@ -32,6 +32,7 @@ import java.io.DataInput; import java.io.IOException; import java.util.List; +import java.util.Map; /** * Table metadata representing a catalog view or a local view from a WITH clause. @@ -65,9 +66,14 @@ public class View extends Table implements GsonPostProcessable, ViewIf { private String inlineViewDef; // for persist + // Replaced by sessionVariables + @Deprecated @SerializedName("sm") private long sqlMode = 0L; + @SerializedName(value = "sv") + private Map sessionVariables; + // Set if this View is from a WITH clause and not persisted in the catalog. private boolean isLocalView; @@ -82,9 +88,9 @@ public View(long id, String name, List schema) { isLocalView = false; } - public void setInlineViewDefWithSqlMode(String inlineViewDef, long sqlMode) { + public void setInlineViewDefWithSessionVariables(String inlineViewDef, Map sessionVariables) { this.inlineViewDef = inlineViewDef; - this.sqlMode = sqlMode; + this.sessionVariables = sessionVariables; } public void setSqlMode(long sqlMode) { @@ -161,4 +167,12 @@ public void resetViewDefForRestore(String srcDbName, String dbName) { public void gsonPostProcess() throws IOException { originalViewDef = ""; } + + public Map getSessionVariables() { + return sessionVariables; + } + + public void setSessionVariables(Map sessionVariables) { + this.sessionVariables = sessionVariables; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 9db79665e061fc..93a7af2d8e095a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -212,7 +212,7 @@ public enum TableFrom { // insert into target tables private final Map, TableIf> insertTargetTables = Maps.newHashMap(); // save view's def and sql mode to avoid them change before lock - private final Map, Pair> viewInfos = Maps.newHashMap(); + private final Map, Pair>> viewInfos = Maps.newHashMap(); // save insert into schema to avoid schema changed between two read locks private final List insertTargetSchema = new ArrayList<>(); @@ -346,18 +346,18 @@ public boolean isHintForcePreAggOn() { * * @return view info, first is view's def sql, second is view's sql mode */ - public Pair getAndCacheViewInfo(List qualifiedViewName, View view) { + public Pair> getAndCacheViewInfo(List qualifiedViewName, View view) { return viewInfos.computeIfAbsent(qualifiedViewName, k -> { String viewDef; - long sqlMode; + Map sessionVariables; view.readLock(); try { viewDef = view.getInlineViewDef(); - sqlMode = view.getSqlMode(); + sessionVariables = view.getSessionVariables(); } finally { view.readUnlock(); } - return Pair.of(viewDef, sqlMode); + return Pair.of(viewDef, sessionVariables); }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java index d150831bd4ccaa..b0bdf41dafc5f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java @@ -133,8 +133,8 @@ private static Map, TableIf> dserializeTables(JSONArray tablesJson) case "VIEW": String viewJsonValue = tableJson.get("TableValue").toString(); newTable = GsonUtils.GSON.fromJson(viewJsonValue, View.class); - ((View) newTable).setInlineViewDefWithSqlMode(tableJson.get("InlineViewDef").toString(), - tableJson.getLong("SqlMode")); + ((View) newTable).setInlineViewDefWithSessionVariables(tableJson.get("InlineViewDef").toString(), + (Map) tableJson.get("SessionVariables")); break; default: newTable = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 57567295c5c60d..6f5b010222ed76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -4727,7 +4727,7 @@ public List withInList(PredicateContext ctx) { public Literal visitDecimalLiteral(DecimalLiteralContext ctx) { try { if (Config.enable_decimal_conversion) { - return new DecimalV3Literal(new BigDecimal(ctx.getText())); + return DecimalV3Literal.createWithCheck256(new BigDecimal(ctx.getText())); } else { return new DecimalLiteral(new BigDecimal(ctx.getText())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 85b7cc1a9dbf25..e05d84da10eb78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -97,6 +97,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalView; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.AutoCloseSessionVariable; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; @@ -109,6 +110,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -586,13 +588,11 @@ private Plan parseAndAnalyzeExternalView( } private Plan parseAndAnalyzeDorisView(View view, List tableQualifier, CascadesContext parentContext) { - Pair viewInfo = parentContext.getStatementContext().getAndCacheViewInfo(tableQualifier, view); - long originalSqlMode = parentContext.getConnectContext().getSessionVariable().getSqlMode(); - parentContext.getConnectContext().getSessionVariable().setSqlMode(viewInfo.second); - try { + Pair> viewInfo = parentContext.getStatementContext() + .getAndCacheViewInfo(tableQualifier, view); + try (AutoCloseSessionVariable autoClose = new AutoCloseSessionVariable(parentContext.getConnectContext(), + viewInfo.second)) { return parseAndAnalyzeView(view, viewInfo.first, parentContext); - } finally { - parentContext.getConnectContext().getSessionVariable().setSqlMode(originalSqlMode); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index cb22abadfdaecb..aba289c640423e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -91,6 +91,7 @@ import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.AutoCloseSessionVariable; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; @@ -464,18 +465,22 @@ private static Map getColumnToOutput( // if processed in upper for loop, will lead to not found slot error // It's the same reason for moving the processing of materialized columns down. for (Column column : generatedColumns) { - GeneratedColumnInfo info = column.getGeneratedColumnInfo(); - Expression parsedExpression = new NereidsParser().parseExpression(info.getExpr().toSqlWithoutTbl()); - Expression boundExpression = new CustomExpressionAnalyzer(boundSink, ctx.cascadesContext, columnToReplaced) - .analyze(parsedExpression); - if (boundExpression instanceof Alias) { - boundExpression = ((Alias) boundExpression).child(); + try (AutoCloseSessionVariable autoClose = new AutoCloseSessionVariable(ctx.connectContext, + column.getSessionVariables())) { + GeneratedColumnInfo info = column.getGeneratedColumnInfo(); + Expression parsedExpression = new NereidsParser().parseExpression(info.getExpr().toSqlWithoutTbl()); + Expression boundExpression = new CustomExpressionAnalyzer(boundSink, ctx.cascadesContext, + columnToReplaced) + .analyze(parsedExpression); + if (boundExpression instanceof Alias) { + boundExpression = ((Alias) boundExpression).child(); + } + boundExpression = ExpressionUtils.replace(boundExpression, replaceMap); + Alias output = new Alias(boundExpression, info.getExprSql()); + columnToOutput.put(column.getName(), output); + columnToReplaced.put(column.getName(), output.toSlot()); + replaceMap.put(output.toSlot(), output.child()); } - boundExpression = ExpressionUtils.replace(boundExpression, replaceMap); - Alias output = new Alias(boundExpression, info.getExprSql()); - columnToOutput.put(column.getName(), output); - columnToReplaced.put(column.getName(), output.toSlot()); - replaceMap.put(output.toSlot(), output.child()); } for (Column column : materializedViewColumn) { if (column.isMaterializedViewColumn()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index 914220f177edbb..bb0965df9c7dbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -49,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.util.RelationUtil; +import org.apache.doris.qe.AutoCloseSessionVariable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -273,14 +274,12 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex } protected void parseAndCollectFromView(List tableQualifier, View view, CascadesContext parentContext) { - Pair viewInfo = parentContext.getStatementContext().getAndCacheViewInfo(tableQualifier, view); - long originalSqlMode = parentContext.getConnectContext().getSessionVariable().getSqlMode(); - parentContext.getConnectContext().getSessionVariable().setSqlMode(viewInfo.second); + Pair> viewInfo = parentContext.getStatementContext() + .getAndCacheViewInfo(tableQualifier, view); LogicalPlan parsedViewPlan; - try { + try (AutoCloseSessionVariable autoClose = new AutoCloseSessionVariable(parentContext.getConnectContext(), + viewInfo.second)) { parsedViewPlan = new NereidsParser().parseSingle(viewInfo.first); - } finally { - parentContext.getConnectContext().getSessionVariable().setSqlMode(originalSqlMode); } if (parsedViewPlan instanceof UnboundResultSink) { parsedViewPlan = (LogicalPlan) ((UnboundResultSink) parsedViewPlan).child(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java index 7c3c4d6c4c615e..1f8dc5ecacaf00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java @@ -429,7 +429,7 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa if (comparisonPredicate instanceof EqualTo) { try { return TypeCoercionUtils.processComparisonPredicate((ComparisonPredicate) - comparisonPredicate.withChildren(left, new DecimalV3Literal( + comparisonPredicate.withChildren(left, DecimalV3Literal.createWithoutCheck256( literal.getValue().setScale(toScale, RoundingMode.UNNECESSARY)))); } catch (ArithmeticException e) { // TODO: the ideal way is to return an If expr like: @@ -443,7 +443,7 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa } else if (comparisonPredicate instanceof NullSafeEqual) { try { return TypeCoercionUtils.processComparisonPredicate((ComparisonPredicate) - comparisonPredicate.withChildren(left, new DecimalV3Literal( + comparisonPredicate.withChildren(left, DecimalV3Literal.createWithoutCheck256( literal.getValue().setScale(toScale, RoundingMode.UNNECESSARY)))); } catch (ArithmeticException e) { return BooleanLiteral.of(false); @@ -625,7 +625,7 @@ private static Optional convertDecimalToSmallerDecimalV3Type(Compari if (literalScale <= leftType.getScale() && literalPrecision - literalScale <= leftType.getRange()) { trailingZerosValue = trailingZerosValue.setScale(leftType.getScale(), RoundingMode.UNNECESSARY); Expression newLiteral = new DecimalV3Literal( - DecimalV3Type.createDecimalV3TypeLooseCheck(leftType.getPrecision(), leftType.getScale()), + DecimalV3Type.createDecimalV3Type(leftType.getPrecision(), leftType.getScale()), trailingZerosValue); return Optional.of(comparisonPredicate.withChildren(left, newLiteral)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java index 63baa76e053dd4..ab8dc6f93a4a5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java @@ -40,10 +40,14 @@ private Add(List children) { super(children, Operator.ADD); } + private Add(Params param) { + super(param); + } + @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Add(children); + return new Add(getParams(children)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java index 54bfe60ad822d5..2d8c5b6bbe1ff5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java @@ -26,21 +26,31 @@ import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.coercion.NumericType; +import org.apache.doris.nereids.util.LazyCompute; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; import java.util.List; +import java.util.function.Supplier; /** * binary arithmetic operator. Such as +, -, *, /. */ public abstract class BinaryArithmetic extends BinaryOperator implements PropagateNullable { + protected final Supplier dataTypeCache; private final Operator legacyOperator; public BinaryArithmetic(List children, Operator legacyOperator) { super(children, legacyOperator.toString()); this.legacyOperator = legacyOperator; + this.dataTypeCache = buildExpressionDataTypeCache(null); + } + + public BinaryArithmetic(Params params) { + super(params.children, params.legacyOperator.toString()); + this.legacyOperator = params.legacyOperator; + this.dataTypeCache = buildExpressionDataTypeCache(params.getOriginDataType()); } public Operator getLegacyOperator() { @@ -52,8 +62,8 @@ public DataType inputType() { return NumericType.INSTANCE; } - @Override - public DataType getDataType() throws UnboundException { + /**computeDataType*/ + public DataType computeDataType() throws UnboundException { DataType t1 = left().getDataType(); DataType t2 = right().getDataType(); if (t1.isDecimalV2Type() && t2.isDecimalV2Type()) { @@ -116,4 +126,21 @@ protected DecimalV3Type processDecimalV3OverFlow(int integralPart, int targetSca } return DecimalV3Type.createDecimalV3Type(precision, scale); } + + private Supplier buildExpressionDataTypeCache(Supplier specifiedDataType) { + if (specifiedDataType != null) { + return specifiedDataType; + } else { + return LazyCompute.of(this::computeDataType); + } + } + + public Params getParams(List children) { + return new Params(this, getLegacyOperator(), children, isInferred()); + } + + @Override + public DataType getDataType() { + return dataTypeCache.get(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java index bf1ce95d1d04d9..60b3ad119de0a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java @@ -46,10 +46,14 @@ private Divide(List children) { super(children, Operator.DIVIDE); } + private Divide(Params param) { + super(param); + } + @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Divide(children); + return new Divide(getParams(children)); } @Override @@ -58,13 +62,13 @@ public R accept(ExpressionVisitor visitor, C context) { } @Override - public DataType getDataType() throws UnboundException { + public DataType computeDataType() throws UnboundException { if (left().getDataType().isDecimalV3Type()) { DecimalV3Type dt1 = (DecimalV3Type) left().getDataType(); DecimalV3Type dt2 = (DecimalV3Type) right().getDataType(); return DecimalV3Type.createDecimalV3Type(dt1.getPrecision(), dt1.getScale() - dt2.getScale()); } - return super.getDataType(); + return super.computeDataType(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java index d152cfe1217d59..f99487d9cc3d78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java @@ -42,10 +42,14 @@ private Mod(List children) { super(children, Operator.MOD); } + private Mod(Params params) { + super(params); + } + @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Mod(children); + return new Mod(getParams(children)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java index bbcd7f4db595b6..24b745afb4863f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java @@ -41,10 +41,14 @@ public Multiply(List children) { super(children, Operator.MULTIPLY); } + private Multiply(Params param) { + super(param); + } + @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Multiply(children); + return new Multiply(getParams(children)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Params.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Params.java new file mode 100644 index 00000000000000..ab633c1f7152fc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Params.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.analysis.ArithmeticExpr.Operator; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.MoreFieldsThread; + +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; + +/**Params*/ +public class Params { + public final Optional originExpression; + public final Operator legacyOperator; + public final List children; + public final boolean inferred; + + public Params(Expression originExpression, Operator legacyOperator, List children, boolean inferred) { + this.originExpression = MoreFieldsThread.isKeepFunctionSignature() + ? Optional.ofNullable(originExpression) : Optional.empty(); + this.legacyOperator = legacyOperator; + this.children = children; + this.inferred = inferred; + } + + public Supplier getOriginDataType() { + if (originExpression.isPresent()) { + return () -> originExpression.get().getDataType(); + } else { + return null; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java index 46e8174a2db808..eec91d7753b42f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java @@ -41,10 +41,14 @@ private Subtract(List children) { super(children, Operator.SUBTRACT); } + private Subtract(Params params) { + super(params); + } + @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Subtract(children); + return new Subtract(getParams(children)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index a6fcab95c32628..e7c23a2a5064cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -518,7 +518,7 @@ public static Expression unixTimestamp(DateV2Literal date) { @ExecFunction(name = "unix_timestamp") public static Expression unixTimestamp(DateTimeV2Literal date) { int scale = date.getDataType().getScale(); - return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(12 + scale, scale), + return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(12 + scale, scale), new BigDecimal(getTimestamp(date.toJavaDateType()))); } @@ -536,7 +536,7 @@ public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral // means the date string doesn't contain time fields. dateObj = LocalDate.parse(date.getValue(), formatter).atStartOfDay(); } - return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(18, 6), + return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(18, 6), new BigDecimal(getTimestamp(dateObj))); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java index 82e55e81f6fd4f..f70da5d8338678 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java @@ -256,7 +256,7 @@ public static Expression multiplyDecimalV3DecimalV3(DecimalV3Literal first, Deci DecimalV3Type t2 = (DecimalV3Type) second.getDataType(); int precision = t1.getPrecision() + t2.getPrecision(); int scale = t1.getScale() + t2.getScale(); - return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(precision, scale), result); + return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(precision, scale), result); } /** @@ -291,11 +291,11 @@ public static Expression divideDecimalV3(DecimalV3Literal first, DecimalV3Litera DecimalV3Type t1 = (DecimalV3Type) first.getDataType(); DecimalV3Type t2 = (DecimalV3Type) second.getDataType(); if (second.getValue().compareTo(BigDecimal.ZERO) == 0) { - return new NullLiteral(DecimalV3Type.createDecimalV3TypeLooseCheck( + return new NullLiteral(DecimalV3Type.createDecimalV3Type( t1.getPrecision(), t1.getScale() - t2.getScale())); } BigDecimal result = first.getValue().divide(second.getValue()); - return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck( + return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type( t1.getPrecision(), t1.getScale() - t2.getScale()), result); } @@ -360,7 +360,8 @@ public static Expression round(DecimalV3Literal first, IntegerLiteral second) { */ @ExecFunction(name = "round") public static Expression round(DoubleLiteral first) { - DecimalV3Literal middleResult = new DecimalV3Literal(new BigDecimal(Double.toString(first.getValue()))); + DecimalV3Literal middleResult = DecimalV3Literal.createWithoutCheck256( + new BigDecimal(Double.toString(first.getValue()))); return new DoubleLiteral(middleResult.round(0).getDouble()); } @@ -369,7 +370,8 @@ public static Expression round(DoubleLiteral first) { */ @ExecFunction(name = "round") public static Expression round(DoubleLiteral first, IntegerLiteral second) { - DecimalV3Literal middleResult = new DecimalV3Literal(new BigDecimal(Double.toString(first.getValue()))); + DecimalV3Literal middleResult = DecimalV3Literal.createWithoutCheck256( + new BigDecimal(Double.toString(first.getValue()))); return new DoubleLiteral(middleResult.round(second.getValue()).getDouble()); } @@ -395,7 +397,8 @@ public static Expression ceil(DecimalV3Literal first, IntegerLiteral second) { */ @ExecFunction(name = "ceil") public static Expression ceil(DoubleLiteral first) { - DecimalV3Literal middleResult = new DecimalV3Literal(new BigDecimal(Double.toString(first.getValue()))); + DecimalV3Literal middleResult = DecimalV3Literal.createWithoutCheck256( + new BigDecimal(Double.toString(first.getValue()))); return new DoubleLiteral(middleResult.roundCeiling(0).getDouble()); } @@ -404,7 +407,8 @@ public static Expression ceil(DoubleLiteral first) { */ @ExecFunction(name = "ceil") public static Expression ceil(DoubleLiteral first, IntegerLiteral second) { - DecimalV3Literal middleResult = new DecimalV3Literal(new BigDecimal(Double.toString(first.getValue()))); + DecimalV3Literal middleResult = DecimalV3Literal.createWithoutCheck256( + new BigDecimal(Double.toString(first.getValue()))); return new DoubleLiteral(middleResult.roundCeiling(second.getValue()).getDouble()); } @@ -430,7 +434,8 @@ public static Expression floor(DecimalV3Literal first, IntegerLiteral second) { */ @ExecFunction(name = "floor") public static Expression floor(DoubleLiteral first) { - DecimalV3Literal middleResult = new DecimalV3Literal(new BigDecimal(Double.toString(first.getValue()))); + DecimalV3Literal middleResult = DecimalV3Literal.createWithoutCheck256( + new BigDecimal(Double.toString(first.getValue()))); return new DoubleLiteral(middleResult.roundFloor(0).getDouble()); } @@ -439,7 +444,8 @@ public static Expression floor(DoubleLiteral first) { */ @ExecFunction(name = "floor") public static Expression floor(DoubleLiteral first, IntegerLiteral second) { - DecimalV3Literal middleResult = new DecimalV3Literal(new BigDecimal(Double.toString(first.getValue()))); + DecimalV3Literal middleResult = DecimalV3Literal.createWithoutCheck256( + new BigDecimal(Double.toString(first.getValue()))); return new DoubleLiteral(middleResult.roundFloor(second.getValue()).getDouble()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java index dcca9bca01d559..72fcf7830a7015 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java @@ -47,7 +47,7 @@ public DecimalV3Literal(BigDecimal value) { * Constructor for DecimalV3Literal */ public DecimalV3Literal(DecimalV3Type dataType, BigDecimal value) { - super(DecimalV3Type.createDecimalV3TypeLooseCheck( + super(DecimalV3Type.createDecimalV3Type( dataType.getPrecision() == -1 ? value.precision() : dataType.getPrecision(), dataType.getScale() == -1 ? value.scale() : dataType.getScale()) ); @@ -62,6 +62,14 @@ public DecimalV3Literal(DecimalV3Type dataType, BigDecimal value) { this.value = Objects.requireNonNull(adjustedValue); } + public static DecimalV3Literal createWithCheck256(BigDecimal value) { + return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(value), value); + } + + public static DecimalV3Literal createWithoutCheck256(BigDecimal value) { + return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeNotCheck256(value), value); + } + @Override public BigDecimal getValue() { return value; @@ -96,7 +104,7 @@ public DecimalV3Literal roundCeiling(int newScale) { if (newScale >= this.getValue().scale()) { return this; } - return new DecimalV3Literal(value.setScale(newScale, RoundingMode.CEILING)); + return createWithoutCheck256(value.setScale(newScale, RoundingMode.CEILING)); } /** @@ -108,7 +116,7 @@ public DecimalV3Literal roundFloor(int newScale) { if (newScale >= this.getValue().scale()) { return this; } - return new DecimalV3Literal(value.setScale(newScale, RoundingMode.FLOOR)); + return createWithoutCheck256(value.setScale(newScale, RoundingMode.FLOOR)); } /** @@ -120,7 +128,7 @@ public DecimalV3Literal round(int newScale) { if (newScale >= this.getValue().scale()) { return this; } - return new DecimalV3Literal(value.setScale(newScale, RoundingMode.HALF_UP)); + return createWithoutCheck256(value.setScale(newScale, RoundingMode.HALF_UP)); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 61a2359266520c..3ba85827e528e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -577,7 +577,7 @@ private static Literal handleDecimalLiteral(ByteBuffer data) throws AnalysisExce String value = new String(bytes); BigDecimal v = new BigDecimal(value); if (Config.enable_decimal_conversion) { - return new DecimalV3Literal(v); + return DecimalV3Literal.createWithCheck256(v); } return new DecimalLiteral(v); } catch (NumberFormatException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 8fcfefa5c644aa..c4463dd897e01f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -40,6 +40,8 @@ import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.qe.SessionVariable; import com.google.common.base.Preconditions; @@ -518,7 +520,9 @@ public Column translateToCatalogStyle() { defaultValue.map(DefaultValue::getValue).orElse(null), onUpdateDefaultValue.isPresent(), onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId, generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null), - generatedColumnsThatReferToThis); + generatedColumnsThatReferToThis, + // 这里所有的列都加上了变量，以后改一下 + ConnectContextUtil.getAffectQueryResultSessionVariables(ConnectContext.get())); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } @@ -533,7 +537,9 @@ public Column translateToCatalogStyleForSchemaChange() { defaultValue.map(DefaultValue::getRawValue).orElse(null), onUpdateDefaultValue.isPresent(), onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId, generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null), - generatedColumnsThatReferToThis); + generatedColumnsThatReferToThis, + // 这里所有的列都加上了变量，以后改一下 + ConnectContextUtil.getAffectQueryResultSessionVariables(ConnectContext.get())); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java index 9d2acba9e6086b..b601aaa9f133ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java @@ -179,7 +179,7 @@ public int getRange() { @Override public DataType conversion() { if (Config.enable_decimal_conversion && shouldConversion) { - return DecimalV3Type.createDecimalV3Type(precision, scale); + return DecimalV3Type.createDecimalV3TypeNotCheck256(precision, scale); } Preconditions.checkArgument(precision > 0 && precision <= MAX_PRECISION, "precision should in (0, " + MAX_PRECISION + "], but real precision is " + precision); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java index b2a664926d3ed7..4b549d7d023a33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.annotation.Developing; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.types.coercion.FractionalType; import org.apache.doris.qe.ConnectContext; @@ -84,14 +83,14 @@ public static DecimalV3Type forType(DataType dataType) { return (DecimalV3Type) dataType; } if (dataType instanceof DecimalV2Type) { - return createDecimalV3Type( + return createDecimalV3TypeNotCheck256( ((DecimalV2Type) dataType).getPrecision(), ((DecimalV2Type) dataType).getScale()); } if (FOR_TYPE_MAP.containsKey(dataType)) { return FOR_TYPE_MAP.get(dataType); } if (dataType.isDateTimeV2Type()) { - return createDecimalV3Type(14 + ((DateTimeV2Type) dataType).getScale(), + return createDecimalV3TypeNotCheck256(14 + ((DateTimeV2Type) dataType).getScale(), ((DateTimeV2Type) dataType).getScale()); } return SYSTEM_DEFAULT; @@ -102,6 +101,12 @@ public static DecimalV3Type createDecimalV3Type(int precision) { return createDecimalV3Type(precision, DEFAULT_SCALE); } + public static DecimalV3Type createDecimalV3Type(BigDecimal bigDecimal) { + int precision = org.apache.doris.analysis.DecimalLiteral.getBigDecimalPrecision(bigDecimal); + int scale = org.apache.doris.analysis.DecimalLiteral.getBigDecimalScale(bigDecimal); + return createDecimalV3Type(precision, scale); + } + /** createDecimalV3Type. */ public static DecimalV3Type createDecimalV3Type(int precision, int scale) { Preconditions.checkArgument(precision > 0 && precision <= MAX_DECIMAL256_PRECISION, @@ -122,42 +127,20 @@ public static DecimalV3Type createDecimalV3Type(int precision, int scale) { } } - public static DecimalV3Type createDecimalV3Type(BigDecimal bigDecimal) { - int precision = org.apache.doris.analysis.DecimalLiteral.getBigDecimalPrecision(bigDecimal); - int scale = org.apache.doris.analysis.DecimalLiteral.getBigDecimalScale(bigDecimal); - return createDecimalV3TypeLooseCheck(precision, scale); + /** createDecimalV3Type. */ + public static DecimalV3Type createDecimalV3TypeNotCheck256(int precision, int scale) { + Preconditions.checkArgument(precision > 0 && precision <= MAX_DECIMAL256_PRECISION, + "precision should in (0, " + MAX_DECIMAL256_PRECISION + "], but real precision is " + precision); + Preconditions.checkArgument(scale >= 0, "scale should not smaller than 0, but real scale is " + scale); + Preconditions.checkArgument(precision >= scale, "precision should not smaller than scale," + + " but precision is " + precision, ", scale is " + scale); + return new DecimalV3Type(precision, scale); } - /** - * create DecimalV3Type, not throwing NotSupportedException. - */ - public static DecimalV3Type createDecimalV3TypeLooseCheck(int precision, int scale) { - boolean enableDecimal256 = false; - ConnectContext connectContext = ConnectContext.get(); - if (connectContext != null) { - enableDecimal256 = connectContext.getSessionVariable().isEnableDecimal256(); - } - if (enableDecimal256) { - if (!(precision > 0 && precision <= MAX_DECIMAL256_PRECISION)) { - throw new AnalysisException( - "precision should in (0, " + MAX_DECIMAL256_PRECISION + "], but real precision is " + precision - ); - } - } else { - if (!(precision > 0 && precision <= MAX_DECIMAL128_PRECISION)) { - throw new AnalysisException( - "precision should in (0, " + MAX_DECIMAL128_PRECISION + "], but real precision is " + precision - ); - } - } - if (scale < 0) { - throw new AnalysisException("scale should not smaller than 0, but real scale is " + scale); - } - if (precision < scale) { - throw new AnalysisException("precision should not smaller than scale," - + " but precision is " + precision + ", scale is " + scale); - } - return new DecimalV3Type(precision, scale); + public static DecimalV3Type createDecimalV3TypeNotCheck256(BigDecimal bigDecimal) { + int precision = org.apache.doris.analysis.DecimalLiteral.getBigDecimalPrecision(bigDecimal); + int scale = org.apache.doris.analysis.DecimalLiteral.getBigDecimalScale(bigDecimal); + return createDecimalV3TypeNotCheck256(precision, scale); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java index 141a35d6834a16..ccbd8bb636b032 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java @@ -29,6 +29,7 @@ import java.io.DataOutput; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Objects; public class AlterViewInfo implements Writable { @@ -38,25 +39,25 @@ public class AlterViewInfo implements Writable { private long tableId; @SerializedName(value = "inlineViewDef") private String inlineViewDef; - @SerializedName(value = "sqlMode") - private long sqlMode; @SerializedName(value = "newFullSchema") private List newFullSchema; @SerializedName(value = "comment") private String comment; + @SerializedName(value = "sv") + private Map session_variables; public AlterViewInfo() { // for persist newFullSchema = Lists.newArrayList(); } - public AlterViewInfo(long dbId, long tableId, String inlineViewDef, List newFullSchema, long sqlMode, - String comment) { + public AlterViewInfo(long dbId, long tableId, String inlineViewDef, List newFullSchema, + Map session_variables, String comment) { this.dbId = dbId; this.tableId = tableId; this.inlineViewDef = inlineViewDef; this.newFullSchema = newFullSchema; - this.sqlMode = sqlMode; + this.session_variables = session_variables; this.comment = comment; } @@ -76,8 +77,8 @@ public List getNewFullSchema() { return newFullSchema; } - public long getSqlMode() { - return sqlMode; + public Map getSessionVariables() { + return session_variables; } public String getComment() { @@ -86,7 +87,7 @@ public String getComment() { @Override public int hashCode() { - return Objects.hash(dbId, tableId, inlineViewDef, sqlMode, newFullSchema); + return Objects.hash(dbId, tableId, inlineViewDef, session_variables, newFullSchema); } @Override @@ -99,7 +100,8 @@ public boolean equals(Object other) { } AlterViewInfo otherInfo = (AlterViewInfo) other; return dbId == otherInfo.getDbId() && tableId == otherInfo.getTableId() - && inlineViewDef.equalsIgnoreCase(otherInfo.getInlineViewDef()) && sqlMode == otherInfo.getSqlMode() + && inlineViewDef.equalsIgnoreCase(otherInfo.getInlineViewDef()) + && session_variables == otherInfo.getSessionVariables() && newFullSchema.equals(otherInfo.getNewFullSchema()) && Objects.equals(comment, otherInfo.comment); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java new file mode 100644 index 00000000000000..81b5dfd70bf04f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import com.google.cloud.hadoop.repackaged.gcs.com.google.common.collect.Maps; + +import java.util.Map; +import java.util.Objects; + +/** + * Used to generate new SessionVariables based on the persisted map, + * and automatically restore to the previous SessionVariables after use. + */ +public class AutoCloseSessionVariable implements AutoCloseable { + public ConnectContext connectContext; + public SessionVariable sessionVariable; + public boolean changed; + + private SessionVariable previousVariable; + + public AutoCloseSessionVariable() { + this.changed = false; + } + + public AutoCloseSessionVariable(ConnectContext connectContext, Map affectQueryResultVariables) { + Objects.requireNonNull(connectContext, "require connectContext object"); + this.changed = true; + this.connectContext = connectContext; + this.previousVariable = connectContext.getSessionVariable(); + sessionVariable = new SessionVariable(); + sessionVariable.setAffectQueryResultSessionVariables( + affectQueryResultVariables == null ? Maps.newHashMap() : affectQueryResultVariables); + connectContext.setSessionVariable(sessionVariable); + } + + public void call() { + // try (AutoSessionVariable autoCloseCtx = new AutoSessionVariable(context)) { + // will report autoCloseCtx is not used, so call an empty method. + } + + @Override + public void close() { + if (changed) { + connectContext.setSessionVariable(previousVariable); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java index 48c7744b570909..165d26c0acf383 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContextUtil.java @@ -27,6 +27,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.nereids.StatementContext; +import java.util.Map; import java.util.Optional; import java.util.stream.Stream; @@ -94,4 +95,11 @@ public static Optional> initCatalogAndDb(ConnectContext ctx.getState().setOk(); return Optional.empty(); } + + public static Map getAffectQueryResultSessionVariables(ConnectContext ctx) { + if (ctx == null || ctx.getSessionVariable() == null) { + return null; + } + return ctx.getSessionVariable().getAffectQueryResultVariables(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index cbfc1d7981ce2b..6a25deb02bf830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5122,30 +5122,9 @@ public void setForwardedSessionVariables(Map variables) { } // set config field - switch (f.getType().getSimpleName()) { - case "short": - f.setShort(this, Short.parseShort(val)); - break; - case "int": - f.setInt(this, Integer.parseInt(val)); - break; - case "long": - f.setLong(this, Long.parseLong(val)); - break; - case "double": - f.setDouble(this, Double.parseDouble(val)); - break; - case "boolean": - f.setBoolean(this, Boolean.parseBoolean(val)); - break; - case "String": - f.set(this, val); - break; - default: - throw new IllegalArgumentException("Unknown field type: " + f.getType().getSimpleName()); - } + VariableMgr.setValue(this, val, f, varAttr.name()); } - } catch (IllegalAccessException e) { + } catch (Throwable e) { LOG.error("failed to set forward variables", e); } } @@ -5794,4 +5773,50 @@ public static boolean isFeDebug() { return false; } } + + public Map getAffectQueryResultVariables() { + HashMap map = new HashMap(); + try { + Field[] fields = SessionVariable.class.getDeclaredFields(); + for (Field f : fields) { + VarAttr varAttr = f.getAnnotation(VarAttr.class); + if (varAttr == null || !varAttr.affectQueryResult() || VariableAnnotation.DEPRECATED.equals( + varAttr.varType()) || VariableAnnotation.REMOVED.equals(varAttr.varType())) { + continue; + } + map.put(varAttr.name(), String.valueOf(f.get(this))); + } + } catch (IllegalAccessException e) { + LOG.error("failed to get affect query result variables", e); + } + return map; + } + + public void setAffectQueryResultSessionVariables(Map variables) { + if (variables.isEmpty()) { + return; + } + try { + Field[] fields = SessionVariable.class.getDeclaredFields(); + for (Field f : fields) { + f.setAccessible(true); + VarAttr varAttr = f.getAnnotation(VarAttr.class); + if (varAttr == null || !varAttr.affectQueryResult() || VariableAnnotation.DEPRECATED.equals( + varAttr.varType()) || VariableAnnotation.REMOVED.equals(varAttr.varType())) { + continue; + } + String val = variables.get(varAttr.name()); + if (val == null) { + continue; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("set affect query result variable: {} = {}", varAttr.name(), val); + } + VariableMgr.setValue(this, val, f, varAttr.name()); + } + } catch (Throwable e) { + LOG.error("failed to set forward variables", e); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index b3d5fc10bd9937..47b0d5f5f2e822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -191,45 +191,7 @@ private static boolean setValue(Object obj, SessionVariableField sessionVariable } } else { try { - switch (field.getType().getSimpleName()) { - case "boolean": - if (value.equalsIgnoreCase("ON") - || value.equalsIgnoreCase("TRUE") - || value.equalsIgnoreCase("1")) { - field.setBoolean(obj, true); - } else if (value.equalsIgnoreCase("OFF") - || value.equalsIgnoreCase("FALSE") - || value.equalsIgnoreCase("0")) { - field.setBoolean(obj, false); - } else { - throw new IllegalAccessException(); - } - break; - case "byte": - field.setByte(obj, Byte.parseByte(value)); - break; - case "short": - field.setShort(obj, Short.parseShort(value)); - break; - case "int": - field.setInt(obj, Integer.parseInt(value)); - break; - case "long": - field.setLong(obj, Long.parseLong(value)); - break; - case "float": - field.setFloat(obj, Float.parseFloat(value)); - break; - case "double": - field.setDouble(obj, Double.parseDouble(value)); - break; - case "String": - field.set(obj, value); - break; - default: - // Unsupported type variable. - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_TYPE_FOR_VAR, attr.name()); - } + setValue(obj, value, field, attr.name()); } catch (NumberFormatException e) { ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_TYPE_FOR_VAR, attr.name()); } catch (IllegalAccessException e) { @@ -244,6 +206,49 @@ private static boolean setValue(Object obj, SessionVariableField sessionVariable return true; } + public static void setValue(Object obj, String value, Field field, String name) + throws IllegalAccessException, DdlException { + switch (field.getType().getSimpleName()) { + case "boolean": + if (value.equalsIgnoreCase("ON") + || value.equalsIgnoreCase("TRUE") + || value.equalsIgnoreCase("1")) { + field.setBoolean(obj, true); + } else if (value.equalsIgnoreCase("OFF") + || value.equalsIgnoreCase("FALSE") + || value.equalsIgnoreCase("0")) { + field.setBoolean(obj, false); + } else { + throw new IllegalAccessException(); + } + break; + case "byte": + field.setByte(obj, Byte.parseByte(value)); + break; + case "short": + field.setShort(obj, Short.parseShort(value)); + break; + case "int": + field.setInt(obj, Integer.parseInt(value)); + break; + case "long": + field.setLong(obj, Long.parseLong(value)); + break; + case "float": + field.setFloat(obj, Float.parseFloat(value)); + break; + case "double": + field.setDouble(obj, Double.parseDouble(value)); + break; + case "String": + field.set(obj, value); + break; + default: + // Unsupported type variable. + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_TYPE_FOR_VAR, name); + } + } + // revert the operator[set_var] on select/*+ SET_VAR()*/ sql; public static void revertSessionValue(SessionVariable obj) throws DdlException { Map sessionOriginValue = obj.getSessionOriginValue(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java index 8a70831c38b6d9..fc842a35b69dca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java @@ -35,6 +35,7 @@ import org.junit.Test; import java.io.File; +import java.util.HashMap; import java.util.UUID; public class CreateViewTest { @@ -222,8 +223,8 @@ public void testViewRejectVarbinary() throws Exception { @Test public void testResetViewDefForRestore() { View view = new View(); - view.setInlineViewDefWithSqlMode("SELECT `internal`.`test`.`test`.`k2` AS `k1`, " - + "FROM `internal`.`test`.`test`;", 1); + view.setInlineViewDefWithSessionVariables("SELECT `internal`.`test`.`test`.`k2` AS `k1`, " + + "FROM `internal`.`test`.`test`;", new HashMap<>()); view.resetViewDefForRestore("test", "test1"); Assert.assertEquals("SELECT `internal`.`test1`.`test`.`k2` AS `k1`, " + "FROM `internal`.`test1`.`test`;", view.getInlineViewDef()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java index 4291d753918cf5..f499caa96f6aeb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -88,6 +88,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; public class OlapQueryCacheTest { @@ -443,7 +444,7 @@ private View createEventView1() { String originStmt = "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `__count_1` FROM `testDb`.`appevent` WHERE ((`eventdate` >= '2020-01-12') AND (`eventdate` <= '2020-01-14')) GROUP BY `eventdate`"; View view = new View(30000L, "view1", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + view.setInlineViewDefWithSessionVariables(originStmt, new HashMap<>()); view.setNewFullSchema(Lists.newArrayList( new Column("eventdate", ScalarType.DATE), new Column("_count_2", ScalarType.BIGINT) @@ -454,7 +455,7 @@ private View createEventView1() { private View createEventView2() { String originStmt = "SELECT `eventdate` AS `eventdate`, `userid` AS `userid` FROM `testDb`.`appevent`"; View view = new View(30001L, "view2", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + view.setInlineViewDefWithSessionVariables(originStmt, new HashMap<>()); view.setNewFullSchema(Lists.newArrayList( new Column("eventdate", ScalarType.DATE), new Column("userid", ScalarType.INT) @@ -466,7 +467,7 @@ private View createEventView3() { String originStmt = "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `count(``userid``)` FROM `testDb`.`appevent` WHERE ((`eventdate` >= '2020-01-12') AND (`eventdate` <= '2020-01-14')) GROUP BY `eventdate`"; View view = new View(30002L, "view3", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + view.setInlineViewDefWithSessionVariables(originStmt, new HashMap<>()); view.setNewFullSchema(Lists.newArrayList( new Column("eventdate", ScalarType.DATE), new Column("_count_2", ScalarType.BIGINT) @@ -478,7 +479,7 @@ private View createEventNestedView() { String originStmt = "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `__count_1` FROM `testDb`.`view2` " + "WHERE ((`eventdate` >= '2020-01-12') AND (`eventdate` <= '2020-01-14')) GROUP BY `eventdate`"; View view = new View(30003L, "view4", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + view.setInlineViewDefWithSessionVariables(originStmt, new HashMap<>()); view.setNewFullSchema(Lists.newArrayList( new Column("eventdate", ScalarType.DATE), new Column("_count_2", ScalarType.BIGINT) From 98452e9450ba6e91d6c4c919849131ff8b34f3cf Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 31 Oct 2025 20:00:00 +0800 Subject: [PATCH 02/58] add sessionvarGuardExpr --- .../glue/translator/ExpressionTranslator.java | 6 + .../nereids/rules/analysis/BindRelation.java | 15 ++- .../nereids/rules/analysis/BindSink.java | 3 + .../doris/nereids/trees/expressions/Add.java | 6 +- .../trees/expressions/BinaryArithmetic.java | 32 +---- .../trees/expressions/BinaryOperator.java | 2 +- .../nereids/trees/expressions/Divide.java | 10 +- .../doris/nereids/trees/expressions/Mod.java | 6 +- .../nereids/trees/expressions/Multiply.java | 6 +- .../expressions/SessionVarGuardExpr.java | 124 ++++++++++++++++++ .../nereids/trees/expressions/Subtract.java | 6 +- .../DateTimeExtractAndTransform.java | 4 +- .../executable/NumericArithmetic.java | 6 +- .../expressions/literal/DecimalV3Literal.java | 2 +- .../visitor/ExpressionVisitor.java | 5 + .../doris/nereids/types/DecimalV3Type.java | 36 ++++- .../apache/doris/persist/AlterViewInfo.java | 12 +- .../doris/qe/AutoCloseSessionVariable.java | 2 +- .../nereids/rules/rewrite/PrepareTest.java | 83 ++++++++++++ .../doris/persist/AlterViewInfoTest.java | 3 +- .../test_generated_column.out | 4 + .../variables_persist/test_view.out | 4 + .../test_alias_function.groovy | 0 .../test_generated_column.groovy | 34 +++++ .../variables_persist/test_view.groovy | 46 +++++++ 25 files changed, 383 insertions(+), 74 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PrepareTest.java create mode 100644 regression-test/data/nereids_p0/variables_persist/test_generated_column.out create mode 100644 regression-test/data/nereids_p0/variables_persist/test_view.out create mode 100644 regression-test/suites/nereids_p0/variables_persist/test_alias_function.groovy create mode 100644 regression-test/suites/nereids_p0/variables_persist/test_generated_column.groovy create mode 100644 regression-test/suites/nereids_p0/variables_persist/test_view.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index f291f4e432f2f2..9515f7c438102b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -76,6 +76,7 @@ import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; import org.apache.doris.nereids.trees.expressions.TryCast; @@ -754,6 +755,11 @@ public Expr visitAggregateExpression(AggregateExpression aggregateExpression, Pl currentPhaseArguments, aggFnArguments, aggregateExpression.getAggregateParam(), context); } + @Override + public Expr visitSessionVarGuardExpr(SessionVarGuardExpr sessionVarGuardExpr, PlanTranslatorContext context) { + return sessionVarGuardExpr.child().accept(this, context); + } + @Override public Expr visitTableGeneratingFunction(TableGeneratingFunction function, PlanTranslatorContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index e05d84da10eb78..e7a90d664fca82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -90,6 +90,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; @@ -590,10 +591,22 @@ private Plan parseAndAnalyzeExternalView( private Plan parseAndAnalyzeDorisView(View view, List tableQualifier, CascadesContext parentContext) { Pair> viewInfo = parentContext.getStatementContext() .getAndCacheViewInfo(tableQualifier, view); + Plan analyzedPlan; try (AutoCloseSessionVariable autoClose = new AutoCloseSessionVariable(parentContext.getConnectContext(), viewInfo.second)) { - return parseAndAnalyzeView(view, viewInfo.first, parentContext); + analyzedPlan = parseAndAnalyzeView(view, viewInfo.first, parentContext); } + // Wrap the analyzed view plan outputs with SessionVarGuardExpr to preserve session variables + if (analyzedPlan instanceof LogicalPlan) { + List guardedProjects = analyzedPlan.getOutput().stream() + .map(NamedExpression.class::cast) + .map(ne -> new Alias( + new org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr(ne, viewInfo.second), + ne.getName())) + .collect(ImmutableList.toImmutableList()); + return new LogicalProject<>(guardedProjects, (LogicalPlan) analyzedPlan); + } + return analyzedPlan; } private Plan parseAndAnalyzeView(TableIf view, String ddlSql, CascadesContext parentContext) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index aba289c640423e..dec459e72d3b82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -476,6 +476,9 @@ private static Map getColumnToOutput( boundExpression = ((Alias) boundExpression).child(); } boundExpression = ExpressionUtils.replace(boundExpression, replaceMap); + // Guard with session variables so subsequent rewrites/merges keep the same decimal policy + boundExpression = new org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr( + boundExpression, column.getSessionVariables()); Alias output = new Alias(boundExpression, info.getExprSql()); columnToOutput.put(column.getName(), output); columnToReplaced.put(column.getName(), output.toSlot()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java index ab8dc6f93a4a5d..63baa76e053dd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java @@ -40,14 +40,10 @@ private Add(List children) { super(children, Operator.ADD); } - private Add(Params param) { - super(param); - } - @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Add(getParams(children)); + return new Add(children); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java index 2d8c5b6bbe1ff5..45c2ea5d4e232d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java @@ -26,31 +26,20 @@ import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.coercion.NumericType; -import org.apache.doris.nereids.util.LazyCompute; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; import java.util.List; -import java.util.function.Supplier; /** * binary arithmetic operator. Such as +, -, *, /. */ public abstract class BinaryArithmetic extends BinaryOperator implements PropagateNullable { - - protected final Supplier dataTypeCache; private final Operator legacyOperator; public BinaryArithmetic(List children, Operator legacyOperator) { super(children, legacyOperator.toString()); this.legacyOperator = legacyOperator; - this.dataTypeCache = buildExpressionDataTypeCache(null); - } - - public BinaryArithmetic(Params params) { - super(params.children, params.legacyOperator.toString()); - this.legacyOperator = params.legacyOperator; - this.dataTypeCache = buildExpressionDataTypeCache(params.getOriginDataType()); } public Operator getLegacyOperator() { @@ -62,8 +51,8 @@ public DataType inputType() { return NumericType.INSTANCE; } - /**computeDataType*/ - public DataType computeDataType() throws UnboundException { + @Override + public DataType getDataType() throws UnboundException { DataType t1 = left().getDataType(); DataType t2 = right().getDataType(); if (t1.isDecimalV2Type() && t2.isDecimalV2Type()) { @@ -126,21 +115,4 @@ protected DecimalV3Type processDecimalV3OverFlow(int integralPart, int targetSca } return DecimalV3Type.createDecimalV3Type(precision, scale); } - - private Supplier buildExpressionDataTypeCache(Supplier specifiedDataType) { - if (specifiedDataType != null) { - return specifiedDataType; - } else { - return LazyCompute.of(this::computeDataType); - } - } - - public Params getParams(List children) { - return new Params(this, getLegacyOperator(), children, isInferred()); - } - - @Override - public DataType getDataType() { - return dataTypeCache.get(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java index 0b8ee94e8b815d..4161acf6649104 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java @@ -56,7 +56,7 @@ public String computeToSql() { @Override public String toString() { - return "(" + left().toString() + " " + symbol + " " + right().toString() + ")"; + return "(" + left().toString() + " " + symbol + " " + right().toString() + ")" + "____type:" + getDataType(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java index 60b3ad119de0a9..bf1ce95d1d04d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java @@ -46,14 +46,10 @@ private Divide(List children) { super(children, Operator.DIVIDE); } - private Divide(Params param) { - super(param); - } - @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Divide(getParams(children)); + return new Divide(children); } @Override @@ -62,13 +58,13 @@ public R accept(ExpressionVisitor visitor, C context) { } @Override - public DataType computeDataType() throws UnboundException { + public DataType getDataType() throws UnboundException { if (left().getDataType().isDecimalV3Type()) { DecimalV3Type dt1 = (DecimalV3Type) left().getDataType(); DecimalV3Type dt2 = (DecimalV3Type) right().getDataType(); return DecimalV3Type.createDecimalV3Type(dt1.getPrecision(), dt1.getScale() - dt2.getScale()); } - return super.computeDataType(); + return super.getDataType(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java index f99487d9cc3d78..d152cfe1217d59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java @@ -42,14 +42,10 @@ private Mod(List children) { super(children, Operator.MOD); } - private Mod(Params params) { - super(params); - } - @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Mod(getParams(children)); + return new Mod(children); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java index 24b745afb4863f..bbcd7f4db595b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java @@ -41,14 +41,10 @@ public Multiply(List children) { super(children, Operator.MULTIPLY); } - private Multiply(Params param) { - super(param); - } - @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Multiply(getParams(children)); + return new Multiply(children); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java new file mode 100644 index 00000000000000..5aae62762e7dc4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.qe.AutoCloseSessionVariable; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A transparent wrapper expression that temporarily applies given session variables + * during type computation and SQL rendering of its child expression. This is used + * to preserve precision/scale decisions for generated columns and alias functions + * that depend on affectQueryResult session variables (e.g. enable_decimal256, decimalOverflowScale). + */ +public class SessionVarGuardExpr extends Expression implements UnaryExpression { + + private final Map sessionVars; + + public SessionVarGuardExpr(Expression child, Map sessionVars) { + // Expose the same children as the wrapped expression to stay transparent for rewrites + super(ImmutableList.of(child)); + this.sessionVars = sessionVars; + } + + public Map getSessionVars() { + return sessionVars; + } + + @Override + public boolean nullable() { + return child().nullable(); + } + + @Override + public DataType getDataType() throws UnboundException { + try (AutoCloseSessionVariable ignored = openGuard()) { + return child().getDataType(); + } + } + + @Override + public String computeToSql() { + try (AutoCloseSessionVariable ignored = openGuard()) { + return child().toSql(); + } + } + + @Override + public Expression withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, "SessionVarGuardExpr must have exactly one child"); + // Rebuild the wrapped expression with provided children, then wrap again + return new SessionVarGuardExpr(children.get(0), sessionVars); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + // Apply session variables during visitor traversal + // try (AutoCloseSessionVariable ignored = openGuard()) { + // return child.accept(visitor, context); + // } + try (AutoCloseSessionVariable ignored = openGuard()) { + return visitor.visitSessionVarGuardExpr(this, context); + } + } + + private AutoCloseSessionVariable openGuard() { + ConnectContext ctx = ConnectContext.get(); + if (ctx == null || sessionVars == null || sessionVars.isEmpty()) { + return new AutoCloseSessionVariable(); + } + return new AutoCloseSessionVariable(ctx, sessionVars); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SessionVarGuardExpr that = (SessionVarGuardExpr) o; + return Objects.equals(child(), that.child()) && Objects.equals(sessionVars, that.sessionVars); + } + + @Override + public int computeHashCode() { + return Objects.hash(super.computeHashCode(), child(), sessionVars); + } + + @Override + public String toString() { + try (AutoCloseSessionVariable ignored = openGuard()) { + return "svGuard:" + child().toString(); + } + } +} + + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java index eec91d7753b42f..46e8174a2db808 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java @@ -41,14 +41,10 @@ private Subtract(List children) { super(children, Operator.SUBTRACT); } - private Subtract(Params params) { - super(params); - } - @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new Subtract(getParams(children)); + return new Subtract(children); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index e7c23a2a5064cf..a6fcab95c32628 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -518,7 +518,7 @@ public static Expression unixTimestamp(DateV2Literal date) { @ExecFunction(name = "unix_timestamp") public static Expression unixTimestamp(DateTimeV2Literal date) { int scale = date.getDataType().getScale(); - return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(12 + scale, scale), + return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(12 + scale, scale), new BigDecimal(getTimestamp(date.toJavaDateType()))); } @@ -536,7 +536,7 @@ public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral // means the date string doesn't contain time fields. dateObj = LocalDate.parse(date.getValue(), formatter).atStartOfDay(); } - return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(18, 6), + return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(18, 6), new BigDecimal(getTimestamp(dateObj))); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java index f70da5d8338678..3aeb33ff09470d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java @@ -256,7 +256,7 @@ public static Expression multiplyDecimalV3DecimalV3(DecimalV3Literal first, Deci DecimalV3Type t2 = (DecimalV3Type) second.getDataType(); int precision = t1.getPrecision() + t2.getPrecision(); int scale = t1.getScale() + t2.getScale(); - return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(precision, scale), result); + return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(precision, scale), result); } /** @@ -291,11 +291,11 @@ public static Expression divideDecimalV3(DecimalV3Literal first, DecimalV3Litera DecimalV3Type t1 = (DecimalV3Type) first.getDataType(); DecimalV3Type t2 = (DecimalV3Type) second.getDataType(); if (second.getValue().compareTo(BigDecimal.ZERO) == 0) { - return new NullLiteral(DecimalV3Type.createDecimalV3Type( + return new NullLiteral(DecimalV3Type.createDecimalV3TypeLooseCheck( t1.getPrecision(), t1.getScale() - t2.getScale())); } BigDecimal result = first.getValue().divide(second.getValue()); - return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type( + return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck( t1.getPrecision(), t1.getScale() - t2.getScale()), result); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java index 72fcf7830a7015..995b24f603bf02 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java @@ -47,7 +47,7 @@ public DecimalV3Literal(BigDecimal value) { * Constructor for DecimalV3Literal */ public DecimalV3Literal(DecimalV3Type dataType, BigDecimal value) { - super(DecimalV3Type.createDecimalV3Type( + super(DecimalV3Type.createDecimalV3TypeLooseCheck( dataType.getPrecision() == -1 ? value.precision() : dataType.getPrecision(), dataType.getScale() == -1 ? value.scale() : dataType.getScale()) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index cf7478bba69a2c..514b1d0d9e0de9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -73,6 +73,7 @@ import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; @@ -173,6 +174,10 @@ public R visitAggregateExpression(AggregateExpression aggregateExpression, C con return visit(aggregateExpression, context); } + public R visitSessionVarGuardExpr(SessionVarGuardExpr sessionVarGuardExpr, C context) { + return visit(sessionVarGuardExpr, context); + } + public R visitAlias(Alias alias, C context) { return visitNamedExpression(alias, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java index 4b549d7d023a33..5c0a95bf0d1eee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV3Type.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.annotation.Developing; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.types.coercion.FractionalType; import org.apache.doris.qe.ConnectContext; @@ -104,7 +105,7 @@ public static DecimalV3Type createDecimalV3Type(int precision) { public static DecimalV3Type createDecimalV3Type(BigDecimal bigDecimal) { int precision = org.apache.doris.analysis.DecimalLiteral.getBigDecimalPrecision(bigDecimal); int scale = org.apache.doris.analysis.DecimalLiteral.getBigDecimalScale(bigDecimal); - return createDecimalV3Type(precision, scale); + return createDecimalV3TypeLooseCheck(precision, scale); } /** createDecimalV3Type. */ @@ -143,6 +144,39 @@ public static DecimalV3Type createDecimalV3TypeNotCheck256(BigDecimal bigDecimal return createDecimalV3TypeNotCheck256(precision, scale); } + /** + * create DecimalV3Type, not throwing NotSupportedException. + */ + public static DecimalV3Type createDecimalV3TypeLooseCheck(int precision, int scale) { + boolean enableDecimal256 = false; + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null) { + enableDecimal256 = connectContext.getSessionVariable().isEnableDecimal256(); + } + if (enableDecimal256) { + if (!(precision > 0 && precision <= MAX_DECIMAL256_PRECISION)) { + throw new AnalysisException( + "precision should in (0, " + MAX_DECIMAL256_PRECISION + "], but real precision is " + precision + ); + } + } else { + if (!(precision > 0 && precision <= MAX_DECIMAL128_PRECISION)) { + throw new AnalysisException( + "precision should in (0, " + MAX_DECIMAL128_PRECISION + "], but real precision is " + precision + ); + } + } + if (scale < 0) { + throw new AnalysisException("scale should not smaller than 0, but real scale is " + scale); + } + if (precision < scale) { + throw new AnalysisException("precision should not smaller than scale," + + " but precision is " + precision + ", scale is " + scale); + } + return new DecimalV3Type(precision, scale); + } + + /** * create DecimalV3Type, without checking precision and scale, e.g. for DataType.fromCatalogType */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java index ccbd8bb636b032..4f5c1df32ab566 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java @@ -44,7 +44,7 @@ public class AlterViewInfo implements Writable { @SerializedName(value = "comment") private String comment; @SerializedName(value = "sv") - private Map session_variables; + private Map sessionVariables; public AlterViewInfo() { // for persist @@ -52,12 +52,12 @@ public AlterViewInfo() { } public AlterViewInfo(long dbId, long tableId, String inlineViewDef, List newFullSchema, - Map session_variables, String comment) { + Map sessionVariables, String comment) { this.dbId = dbId; this.tableId = tableId; this.inlineViewDef = inlineViewDef; this.newFullSchema = newFullSchema; - this.session_variables = session_variables; + this.sessionVariables = sessionVariables; this.comment = comment; } @@ -78,7 +78,7 @@ public List getNewFullSchema() { } public Map getSessionVariables() { - return session_variables; + return sessionVariables; } public String getComment() { @@ -87,7 +87,7 @@ public String getComment() { @Override public int hashCode() { - return Objects.hash(dbId, tableId, inlineViewDef, session_variables, newFullSchema); + return Objects.hash(dbId, tableId, inlineViewDef, sessionVariables, newFullSchema); } @Override @@ -101,7 +101,7 @@ public boolean equals(Object other) { AlterViewInfo otherInfo = (AlterViewInfo) other; return dbId == otherInfo.getDbId() && tableId == otherInfo.getTableId() && inlineViewDef.equalsIgnoreCase(otherInfo.getInlineViewDef()) - && session_variables == otherInfo.getSessionVariables() + && sessionVariables == otherInfo.getSessionVariables() && newFullSchema.equals(otherInfo.getNewFullSchema()) && Objects.equals(comment, otherInfo.comment); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java index 81b5dfd70bf04f..50961e1038dbe6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java @@ -17,7 +17,7 @@ package org.apache.doris.qe; -import com.google.cloud.hadoop.repackaged.gcs.com.google.common.collect.Maps; +import com.google.common.collect.Maps; import java.util.Map; import java.util.Objects; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PrepareTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PrepareTest.java new file mode 100644 index 00000000000000..3a84779aa8e905 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PrepareTest.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class PrepareTest { + @Test + public void prepare() { + // 数据库连接信息 + // String url = "jdbc:mysql://127.0.0.1:9030/maldb?useServerPrepStmts=true"; + String url = "jdbc:mysql://127.0.0.1:9030/maldb?rewriteBatchedStatements=true&defaultFetchSize=500&useCursorFetch=true"; + String username = "root"; + String password = ""; + + // String sql = "with cte1 as ( select cast ( date_trunc ( col_date_undef_signed_not_null , ? ) as datetime ) col_alias43212 , min_by ( col_datetime_6__undef_signed_not_null , pk ) col_alias43213 from table_200_undef_partitions2_keys3_properties4_distributed_by55 group by col_alias43212 order by col_alias43212 , col_alias43213 ) , cte2 as ( select col_alias43213 from cte1 where col_alias43213 < ? ) select * from cte2"; + // String sql = "insert into t4 values (?, cast(? as varchar(3)), ?)"; + // String sql = "insert into t4 values (?, ?, ?)"; + // String sql = "select v1 from t4 where `key`=?"; + // String sql = " select min ( pk - ? ) pk , pk as pk from table_20_undef_partitions2_keys3_properties4_distributed_by54 tbl_alias1 group by pk having ( pk >= pk ) or ( round ( sign ( sign ( pk ) ) ) - ? < ? ) order by pk "; + // String sql = "select (? + col_largeint_undef_signed_not_null) col_alias27416 , col_largeint_undef_signed AS col_alias27417 , min_by (col_largeint_undef_signed_not_null, pk) AS col_alias27418 from table_200_undef_partitions2_keys3_properties4_distributed_by53 GROUP BY col_alias27416,col_alias27417 having col_alias27416 = col_alias27418 ORDER BY col_alias27416,col_alias27417,col_alias27418 ;"; + // String sql = "select least(col_largeint_undef_signed, ?, col_smallint_undef_signed_not_null, col_smallint_undef_signed_not_null) col_alias39343 from table_200_undef_partitions2_keys3_properties4_distributed_by54"; + // String sql = "select cast ( date_trunc ( col_datetime_undef_signed_not_null , ? ) as datetime ) col_alias5871 from table_200_undef_partitions2_keys3_properties4_distributed_by52 order by col_alias5871"; + // String sql = "select col_struct_c_char_255_char_255___c_char_100_char_100___c_varchar_255_varchar_255___c_varchar_65533_varchar_65533___c_string_string__undef_signed_not_null , max ( col_char_255__undef_signed ) over ( order by pk rows between ? preceding and current row ) as col_alias27416 from table_20_undef_partitions2_keys3_properties4_distributed_by56 where elt ( ? , col_char_100__undef_signed , struct_element ( col_struct_c_char_255_char_255___c_char_100_char_100___c_varchar_255_varchar_255___c_varchar_65533_varchar_65533___c_string_string__undef_signed , ? ) , col_varchar_65533__undef_signed ) = ?"; + // String sql = "SELECT IFNULL(col_date, ?) FROM t3 where pk = ?"; + // String sql = "select array_apply([1,2,3], ?, ?)"; + String sql = "select ? from t4"; + try { + // 显式加载 MySQL JDBC 驱动程序 + Class.forName("com.mysql.cj.jdbc.Driver"); + // 或者对于旧版本的 MySQL 驱动： + // Class.forName("com.mysql.jdbc.Driver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + return; + } + try (Connection connection = DriverManager.getConnection(url, username, password)) { + PreparedStatement stmt = connection.prepareStatement(sql); + + // stmt.setInt(1, 2); + // stmt.setString(2, "aabsssssswerwerwerwwers"); + // stmt.setString(3, "2"); + // stmt.execute(); + // stmt.setString(1, "3"); + // stmt.setString(2, "aabcgaaa"); + // stmt.setString(3, "3"); + // stmt.execute(); + + // stmt.setBigDecimal(1, BigDecimal.valueOf(100.2)); + stmt.setString(1, "="); + // stmt.setInt(2, 1); + // stmt.setInt(2, 0); + ResultSet resultSet = stmt.executeQuery(); + while (resultSet.next()) { + // 根据列索引获取数据 + System.out.println(resultSet.getString(1)); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterViewInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterViewInfoTest.java index b8e8044d22e95f..8ca5573c2acea0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterViewInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterViewInfoTest.java @@ -32,6 +32,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.HashMap; public class AlterViewInfoTest { private static String fileName = "./AlterViewInfoTest"; @@ -57,7 +58,7 @@ public void testSerializeAlterViewInfo() throws IOException, AnalysisException { Column column1 = new Column("col1", PrimitiveType.BIGINT); Column column2 = new Column("col2", PrimitiveType.DOUBLE); - AlterViewInfo alterViewInfo = new AlterViewInfo(dbId, tableId, inlineViewDef, Lists.newArrayList(column1, column2), sqlMode, null); + AlterViewInfo alterViewInfo = new AlterViewInfo(dbId, tableId, inlineViewDef, Lists.newArrayList(column1, column2), new HashMap<>(), null); alterViewInfo.write(out); out.flush(); out.close(); diff --git a/regression-test/data/nereids_p0/variables_persist/test_generated_column.out b/regression-test/data/nereids_p0/variables_persist/test_generated_column.out new file mode 100644 index 00000000000000..6a7d3f900e26b2 --- /dev/null +++ b/regression-test/data/nereids_p0/variables_persist/test_generated_column.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !c_scale_is_11 -- +1.12343 1.123457 1.26212529751 + diff --git a/regression-test/data/nereids_p0/variables_persist/test_view.out b/regression-test/data/nereids_p0/variables_persist/test_view.out new file mode 100644 index 00000000000000..ef8529eb72ac28 --- /dev/null +++ b/regression-test/data/nereids_p0/variables_persist/test_view.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !scale_is_11 -- +999999999999998246906000000000.76833464320 + diff --git a/regression-test/suites/nereids_p0/variables_persist/test_alias_function.groovy b/regression-test/suites/nereids_p0/variables_persist/test_alias_function.groovy new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/regression-test/suites/nereids_p0/variables_persist/test_generated_column.groovy b/regression-test/suites/nereids_p0/variables_persist/test_generated_column.groovy new file mode 100644 index 00000000000000..78eb9c6a714b34 --- /dev/null +++ b/regression-test/suites/nereids_p0/variables_persist/test_generated_column.groovy @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column") { + // 打开enable_decimal256建表 + multi_sql """ + set enable_decimal256=true; + drop table if exists t_gen_col_multi_decimalv3; + create table t_gen_col_multi_decimalv3(a decimal(20,5),b decimal(21,6),c decimal(38,11) generated always as (a*b) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + // 关闭enable_decimal256，插入数据 + sql "set enable_decimal256=false;" + sql "insert into t_gen_col_multi_decimalv3 values(1.12343,1.123457,default);" + + // 查询数据,预期column c的scale为11 + qt_c_scale_is_11 "select * from t_gen_col_multi_decimalv3;" + +} \ No newline at end of file diff --git a/regression-test/suites/nereids_p0/variables_persist/test_view.groovy b/regression-test/suites/nereids_p0/variables_persist/test_view.groovy new file mode 100644 index 00000000000000..b080c7a612b725 --- /dev/null +++ b/regression-test/suites/nereids_p0/variables_persist/test_view.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_view") { + + multi_sql """drop table if exists test_decimal_mul_overflow1; + CREATE TABLE `test_decimal_mul_overflow1` ( + `f1` decimal(20,5) NULL, + `f2` decimal(21,6) NULL + )DISTRIBUTED BY HASH(f1) + PROPERTIES("replication_num" = "1"); + insert into test_decimal_mul_overflow1 values(999999999999999.12345,999999999999999.123456);""" + + // 这个结果是完整的结果，超出了38个精度 + // 999999999999998246906000000000.76833464320 + // 这个是截断到38个精度的结果 + // 999999999999998246906000000000.76833464 + + // 打开enable256,创建视图; + sql "set enable_decimal256=true;" + sql "drop view if exists v_test_decimal_mul_overflow1;" + sql """create view v_test_decimal_mul_overflow1 as select f1,f2,f1*f2 multi from test_decimal_mul_overflow1;""" + + // 关闭enable256,进行查询，预期结果是multi超出38个精度的结果, multi+1仍然是38个精度 + sql "set enable_decimal256=false;" + + // expect column multi scale is 11:999999999999998246906000000000.76833464320 instead of 8: 999999999999998246906000000000.76833464 + qt_scale_is_11 "select multi from v_test_decimal_mul_overflow1;" + // expect column c1 scale is 8: 999999999999998246906000000000.76833464 +// qt_scale_is_8 "select multi+1 c1 from v_test_decimal_mul_overflow1;" + +} \ No newline at end of file From 193e6e057c2c9e1be96f2c7f72d0f01c0f3fde15 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 4 Nov 2025 10:45:01 +0800 Subject: [PATCH 03/58] add print --- .../apache/doris/nereids/rules/analysis/BindRelation.java | 7 ++++++- .../org/apache/doris/nereids/trees/expressions/Alias.java | 2 +- .../doris/nereids/trees/expressions/SlotReference.java | 4 ++-- .../trees/expressions/functions/agg/AggregateFunction.java | 3 ++- .../data/nereids_p0/variables_persist/test_view.out | 3 +++ .../suites/nereids_p0/variables_persist/test_view.groovy | 2 +- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index e7a90d664fca82..8b78b3f8977727 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -96,6 +96,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan; import org.apache.doris.nereids.trees.plans.logical.LogicalView; +import org.apache.doris.nereids.trees.plans.logical.ProjectMergeable; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.AutoCloseSessionVariable; @@ -604,7 +605,11 @@ private Plan parseAndAnalyzeDorisView(View view, List tableQualifier, Ca new org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr(ne, viewInfo.second), ne.getName())) .collect(ImmutableList.toImmutableList()); - return new LogicalProject<>(guardedProjects, (LogicalPlan) analyzedPlan); + LogicalProject newProject = new LogicalProject<>(guardedProjects, (LogicalPlan) analyzedPlan); + if (analyzedPlan instanceof ProjectMergeable) { + return ProjectMergeable.mergeContinuedProjects(guardedProjects, analyzedPlan).orElse(newProject); + } + return newProject; } return analyzedPlan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 0650a7cbf86f05..b3058f73829040 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -151,7 +151,7 @@ public int computeHashCode() { @Override public String toString() { - return child().toString() + " AS `" + name.get() + "`#" + exprId; + return child().toString() + " AS `" + name.get() + "`#" + exprId + "___" + getDataType(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 1c77dc669acb72..59e7486b9c7139 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -216,9 +216,9 @@ public String computeToSql() { public String toString() { if (subPath.isEmpty()) { // Just return name and exprId, add another method to show fully qualified name when it's necessary. - return name.get() + "#" + exprId; + return name.get() + "#" + exprId + "___" + getDataType(); } - return name.get() + "['" + String.join("']['", subPath) + "']" + "#" + exprId; + return name.get() + "['" + String.join("']['", subPath) + "']" + "#" + exprId + "___" + getDataType(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index e356811fdefc47..8787f9632e92ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -159,7 +159,8 @@ public String toString() { .stream() .map(Expression::toString) .collect(Collectors.joining(", ")); - return getName() + "(" + (distinct ? "DISTINCT " : "") + args + ")"; + return getName() + "(" + (distinct ? "DISTINCT " : "") + args + ")" + "___" + getDataType() + + "___sig:" + getSignature(); } @Override diff --git a/regression-test/data/nereids_p0/variables_persist/test_view.out b/regression-test/data/nereids_p0/variables_persist/test_view.out index ef8529eb72ac28..58ef6e1abc90ef 100644 --- a/regression-test/data/nereids_p0/variables_persist/test_view.out +++ b/regression-test/data/nereids_p0/variables_persist/test_view.out @@ -2,3 +2,6 @@ -- !scale_is_11 -- 999999999999998246906000000000.76833464320 +-- !scale_is_8 -- +999999999999998246906000000001.76833464 + diff --git a/regression-test/suites/nereids_p0/variables_persist/test_view.groovy b/regression-test/suites/nereids_p0/variables_persist/test_view.groovy index b080c7a612b725..32cb216d0a675c 100644 --- a/regression-test/suites/nereids_p0/variables_persist/test_view.groovy +++ b/regression-test/suites/nereids_p0/variables_persist/test_view.groovy @@ -41,6 +41,6 @@ suite("test_view") { // expect column multi scale is 11:999999999999998246906000000000.76833464320 instead of 8: 999999999999998246906000000000.76833464 qt_scale_is_11 "select multi from v_test_decimal_mul_overflow1;" // expect column c1 scale is 8: 999999999999998246906000000000.76833464 -// qt_scale_is_8 "select multi+1 c1 from v_test_decimal_mul_overflow1;" + qt_scale_is_8 "select multi+1 c1 from v_test_decimal_mul_overflow1;" } \ No newline at end of file From 2feb10d0520fd766237b8eabd8072b8c1130cefd Mon Sep 17 00:00:00 2001 From: jacktengg Date: Tue, 4 Nov 2025 10:54:00 +0800 Subject: [PATCH 04/58] test --- be/src/vec/aggregate_functions/aggregate_function_sum.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp index 768d4e5df5b92c..d8d8cd1fb7f174 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp @@ -36,6 +36,7 @@ void register_aggregate_function_sum(AggregateFunctionSimpleFactory& factory) { creator(name, types, result_is_nullable, attr); } else { + // use types of fe return creator_with_type_list< TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, From e002d12671f6ea79f1acd954e4a8575cc8a932a9 Mon Sep 17 00:00:00 2001 From: jacktengg Date: Tue, 4 Nov 2025 10:54:00 +0800 Subject: [PATCH 05/58] use datatype from FE for decimal sum --- be/src/runtime/primitive_type.h | 9 + .../aggregate_function_sum.cpp | 19 +- .../aggregate_function_sum.h | 182 +++++++++++++++--- be/src/vec/aggregate_functions/helpers.h | 63 +++++- be/src/vec/exprs/vectorized_agg_fn.cpp | 3 +- 5 files changed, 240 insertions(+), 36 deletions(-) diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index 6dd0fed82be38f..282996520e8a06 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -227,6 +227,15 @@ constexpr bool is_decimal(PrimitiveType type) { type == TYPE_DECIMAL256 || type == TYPE_DECIMALV2; } +constexpr bool is_decimalv3(PrimitiveType type) { + return type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128I || + type == TYPE_DECIMAL256; +} + +constexpr bool is_same_or_wider_decimalv3(PrimitiveType type1, PrimitiveType type2) { + return is_decimalv3(type1) && is_decimalv3(type2) && (type2 >= type1); +} + constexpr bool is_number(PrimitiveType type) { return is_int_or_bool(type) || is_float_or_double(type) || is_decimal(type); } diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp index d8d8cd1fb7f174..72f9ab22f6e9cb 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp @@ -20,6 +20,7 @@ #include "vec/aggregate_functions/aggregate_function_sum.h" +#include "runtime/primitive_type.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" @@ -28,20 +29,20 @@ namespace doris::vectorized { void register_aggregate_function_sum(AggregateFunctionSimpleFactory& factory) { AggregateFunctionCreator creator = [&](const std::string& name, const DataTypes& types, + const DataTypePtr& result_type, const bool result_is_nullable, const AggregateFunctionAttr& attr) { - if (attr.enable_decimal256 && is_decimal(types[0]->get_primitive_type())) { + if (is_decimalv3(types[0]->get_primitive_type())) { + // use types of fe return creator_with_type_list:: - creator(name, types, result_is_nullable, - attr); + creator_with_result_type( + name, types, result_type, result_is_nullable, attr); } else { - // use types of fe - return creator_with_type_list< - TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, - TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, - TYPE_DECIMALV2>::creator(name, types, - result_is_nullable, attr); + return creator_with_type_list:: + creator(name, types, result_type, + result_is_nullable, attr); } }; factory.register_function_both("sum", creator); diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.h b/be/src/vec/aggregate_functions/aggregate_function_sum.h index 4151316f447a87..b7457111b9cd8f 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sum.h +++ b/be/src/vec/aggregate_functions/aggregate_function_sum.h @@ -25,6 +25,8 @@ #include #include +#include "common/exception.h" +#include "runtime/primitive_type.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column.h" #include "vec/common/assert_cast.h" @@ -66,9 +68,151 @@ struct AggregateFunctionSumData { typename PrimitiveTypeTraits::ColumnItemType get() const { return sum; } }; +template +class AggregateFunctionSum + : public IAggregateFunctionDataHelper>, + UnaryExpression, + NullableAggregateFunction { +public: + AggregateFunctionSum(const DataTypes& argument_types_) + : IAggregateFunctionDataHelper>( + argument_types_) {} + String get_name() const override { return "sum"; } + DataTypePtr get_return_type() const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement get_return_type of AggregateFunctionSum"); + } + void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num, + Arena&) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement get_return_type of AggregateFunctionSum"); + } + void reset(AggregateDataPtr place) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement reset of AggregateFunctionSum"); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, + Arena&) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement merge of AggregateFunctionSum"); + } + + void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement serialize of AggregateFunctionSum"); + } + + void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, + Arena&) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement deserialize of AggregateFunctionSum"); + } + + void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement insert_result_into of AggregateFunctionSum"); + } + + void deserialize_from_column(AggregateDataPtr places, const IColumn& column, Arena&, + size_t num_rows) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement deserialize_from_column of AggregateFunctionSum"); + } + + void serialize_to_column(const std::vector& places, size_t offset, + MutableColumnPtr& dst, const size_t num_rows) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement serialize_to_column of AggregateFunctionSum"); + } + + void streaming_agg_serialize_to_column(const IColumn** columns, MutableColumnPtr& dst, + const size_t num_rows, Arena&) const override { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "not implement streaming_agg_serialize_to_column of AggregateFunctionSum"); + } + + void deserialize_and_merge_from_column(AggregateDataPtr __restrict place, const IColumn& column, + Arena&) const override { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "not implement deserialize_and_merge_from_column of AggregateFunctionSum"); + } + + void deserialize_and_merge_from_column_range(AggregateDataPtr __restrict place, + const IColumn& column, size_t begin, size_t end, + Arena&) const override { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "not implement deserialize_and_merge_from_column_range of AggregateFunctionSum"); + } + + void deserialize_and_merge_vec(const AggregateDataPtr* places, size_t offset, + AggregateDataPtr rhs, const IColumn* column, Arena& arena, + const size_t num_rows) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement deserialize_and_merge_vec of AggregateFunctionSum"); + } + + void deserialize_and_merge_vec_selected(const AggregateDataPtr* places, size_t offset, + AggregateDataPtr rhs, const IColumn* column, + Arena& arena, const size_t num_rows) const override { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "not implement deserialize_and_merge_vec_selected of AggregateFunctionSum"); + } + + void serialize_without_key_to_column(ConstAggregateDataPtr __restrict place, + IColumn& to) const override { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "not implement serialize_without_key_to_column of AggregateFunctionSum"); + } + + MutableColumnPtr create_serialize_column() const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement create_serialize_column of AggregateFunctionSum"); + } + + DataTypePtr get_serialized_type() const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement get_serialized_type of AggregateFunctionSum"); + } + + bool supported_incremental_mode() const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement supported_incremental_mode of AggregateFunctionSum"); + } + + NO_SANITIZE_UNDEFINED void execute_function_with_incremental( + int64_t partition_start, int64_t partition_end, int64_t frame_start, int64_t frame_end, + AggregateDataPtr place, const IColumn** columns, Arena& arena, bool previous_is_nul, + bool end_is_nul, bool has_null, UInt8* use_null_result, + UInt8* could_use_previous_result) const override { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "not implement execute_function_with_incremental of AggregateFunctionSum"); + } + + void add_range_single_place(int64_t partition_start, int64_t partition_end, int64_t frame_start, + int64_t frame_end, AggregateDataPtr place, const IColumn** columns, + Arena& arena, UInt8* use_null_result, + UInt8* could_use_previous_result) const override { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "not implement add_range_single_place of AggregateFunctionSum"); + } +}; + +template +constexpr static bool is_valid_sum_types = + (is_same_or_wider_decimalv3(T, TResult) || + (is_float_or_double(T) && is_float_or_double(TResult)) || + (is_int_or_bool(T) && is_int_or_bool(TResult))); /// Counts the sum of the numbers. template -class AggregateFunctionSum final + requires(is_valid_sum_types) +class AggregateFunctionSum final : public IAggregateFunctionDataHelper>, UnaryExpression, NullableAggregateFunction { @@ -85,7 +229,7 @@ class AggregateFunctionSum final scale(get_decimal_scale(*argument_types_[0])) {} DataTypePtr get_return_type() const override { - if constexpr (is_decimal(T)) { + if constexpr (is_decimal(TResult)) { return std::make_shared(ResultDataType::max_precision(), scale); } else { return std::make_shared(); @@ -275,41 +419,35 @@ class AggregateFunctionSum final UInt32 scale; }; -template +template struct SumSimple { + static_assert(!is_decimalv3(T)); /// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64 static constexpr PrimitiveType ResultType = - level_up ? (T == TYPE_DECIMALV2 - ? TYPE_DECIMALV2 - : (is_decimal(T) ? TYPE_DECIMAL128I - : PrimitiveTypeTraits::NearestPrimitiveType)) - : T; + T == TYPE_DECIMALV2 ? TYPE_DECIMALV2 : PrimitiveTypeTraits::NearestPrimitiveType; using AggregateDataType = AggregateFunctionSumData; using Function = AggregateFunctionSum; }; template -using AggregateFunctionSumSimple = typename SumSimple::Function; +using AggregateFunctionSumSimple = typename SumSimple::Function; -template -struct SumSimpleDecimal256 { - /// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64 - static constexpr PrimitiveType ResultType = - level_up ? (T == TYPE_DECIMALV2 - ? TYPE_DECIMALV2 - : (is_decimal(T) ? TYPE_DECIMAL256 - : PrimitiveTypeTraits::NearestPrimitiveType)) - : T; +template +struct SumSimpleNew { using AggregateDataType = AggregateFunctionSumData; - using Function = AggregateFunctionSum; + using Function = AggregateFunctionSum; }; +template +using AggregateFunctionSumSimpleNew = typename SumSimpleNew::Function; template -using AggregateFunctionSumSimpleDecimal256 = typename SumSimpleDecimal256::Function; - +struct SumSimpleForAggReader { + using AggregateDataType = AggregateFunctionSumData; + using Function = AggregateFunctionSum; +}; // do not level up return type for agg reader template -using AggregateFunctionSumSimpleReader = typename SumSimple::Function; +using AggregateFunctionSumSimpleReader = typename SumSimpleForAggReader::Function; } // namespace doris::vectorized diff --git a/be/src/vec/aggregate_functions/helpers.h b/be/src/vec/aggregate_functions/helpers.h index fd62bf1644ad09..dcfc29b4cb28d3 100644 --- a/be/src/vec/aggregate_functions/helpers.h +++ b/be/src/vec/aggregate_functions/helpers.h @@ -20,8 +20,10 @@ #pragma once +#include "runtime/define_primitive_type.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/aggregate_functions/aggregate_function_null.h" +#include "vec/core/call_on_type_index.h" #include "vec/data_types/data_type.h" #include "vec/utils/template_helpers.hpp" @@ -229,10 +231,10 @@ struct creator_without_type { const bool result_is_nullable, const AggregateFunctionAttr& attr, TArgs&&... args) { - if (!(argument_types_.size() == 1)) { - throw doris::Exception(Status::InternalError( - "create_unary_arguments: argument_types_ size must be 1")); - } + // if (!(argument_types_.size() == 1)) { + // throw doris::Exception(Status::InternalError( + // "create_unary_arguments: argument_types_ size must be 1")); + // } std::unique_ptr result(std::make_unique( std::forward(args)..., remove_nullable(argument_types_))); if (have_nullable(argument_types_)) { @@ -291,6 +293,11 @@ struct CurryDirect { template using T = AggregateFunctionTemplate; }; +template