From 8d7bb266fc9ec327fdfe8989b018d564bc239c7c Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Wed, 26 Jul 2023 16:09:05 +0800 Subject: [PATCH 1/4] [imporve](flex) support scientific notation(aEb) parser --- fe/fe-core/src/main/jflex/sql_scanner.flex | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 62e7c6f3397f88..53b3bdb7663091 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -778,16 +778,5 @@ EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminato return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext()); } -{DoubleLiteral} { - BigDecimal decimal_val; - try { - decimal_val = new BigDecimal(yytext()); - } catch (NumberFormatException e) { - return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext()); - } - - return newToken(SqlParserSymbols.DECIMAL_LITERAL, decimal_val); -} - {Comment} { /* ignore */ } {Whitespace} { /* ignore */ } From 1bb6a5695acd3439b65034dd47a1c3765b40c48b Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Wed, 26 Jul 2023 17:17:54 +0800 Subject: [PATCH 2/4] update --- fe/fe-core/src/main/jflex/sql_scanner.flex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 53b3bdb7663091..62e7c6f3397f88 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -778,5 +778,16 @@ EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminato return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext()); } +{DoubleLiteral} { + BigDecimal decimal_val; + try { + decimal_val = new BigDecimal(yytext()); + } catch (NumberFormatException e) { + return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext()); + } + + return newToken(SqlParserSymbols.DECIMAL_LITERAL, decimal_val); +} + {Comment} { /* ignore */ } {Whitespace} { /* ignore */ } From 44e3c0c5d9faeafeca53a4f98ffdab2dd9bb8ffc Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Mon, 31 Jul 2023 17:24:26 +0800 Subject: [PATCH 3/4] [Test](function) improve if function exectue time when else if null --- be/src/vec/functions/if.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/be/src/vec/functions/if.cpp b/be/src/vec/functions/if.cpp index bcdf147b1db4d1..c95b6b93c854fd 100644 --- a/be/src/vec/functions/if.cpp +++ b/be/src/vec/functions/if.cpp @@ -499,12 +499,12 @@ class FunctionIf : public IFunction { then_column.column = materialize_column_if_const(then_column.column); } } - if (auto* else_is_const = check_and_get_column(*arg_else.column)) { - if (check_and_get_column(else_is_const->get_data_column())) { - ColumnWithTypeAndName& else_column = block.get_by_position(arguments[2]); - else_column.column = materialize_column_if_const(else_column.column); - } - } + // if (auto* else_is_const = check_and_get_column(*arg_else.column)) { + // if (check_and_get_column(else_is_const->get_data_column())) { + // ColumnWithTypeAndName& else_column = block.get_by_position(arguments[2]); + // else_column.column = materialize_column_if_const(else_column.column); + // } + // } Status ret = Status::OK(); if (execute_for_null_condition(context, block, arg_cond, arg_then, arg_else, result) || From 0a06f532b04dba88fe835a2bdb0d025e28f0f635 Mon Sep 17 00:00:00 2001 From: zhangstar333 <2561612514@qq.com> Date: Tue, 1 Aug 2023 17:09:19 +0800 Subject: [PATCH 4/4] update if --- be/src/vec/functions/if.cpp | 69 ++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/be/src/vec/functions/if.cpp b/be/src/vec/functions/if.cpp index c95b6b93c854fd..a07f6c51923b89 100644 --- a/be/src/vec/functions/if.cpp +++ b/be/src/vec/functions/if.cpp @@ -157,11 +157,13 @@ class FunctionIf : public IFunction { } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - DataTypePtr type = nullptr; - get_least_supertype(DataTypes {arguments[1], arguments[2]}, &type); - DCHECK_NE(type, nullptr) << " arguments[1]: " << arguments[1]->get_name() - << " arguments[2]: " << arguments[2]->get_name(); - return type; + // if return type is custom, one of nullable return type will be nullable + bool nullable = arguments[1]->is_nullable() || arguments[2]->is_nullable(); + if (nullable) { + return make_nullable(arguments[1]); + } else { + return arguments[1]; + } } static ColumnPtr materialize_column_if_const(const ColumnPtr& column) { @@ -293,7 +295,9 @@ class FunctionIf : public IFunction { bool then_is_null = arg_then.column->only_null(); bool else_is_null = arg_else.column->only_null(); - if (!then_is_null && !else_is_null) return false; + if (!then_is_null && !else_is_null) { + return false; + } if (then_is_null && else_is_null) { block.get_by_position(result).column = @@ -309,24 +313,24 @@ class FunctionIf : public IFunction { /// If then is NULL, we create Nullable column with null mask OR-ed with condition. if (then_is_null) { if (cond_col) { - if (is_column_nullable(*arg_else.column)) { + if (is_column_nullable(*arg_else.column)) { // if(cond, null, nullable) auto arg_else_column = arg_else.column; auto result_column = (*std::move(arg_else_column)).mutate(); assert_cast(*result_column) .apply_null_map(assert_cast(*arg_cond.column)); block.replace_by_position(result, std::move(result_column)); - } else { + } else { // if(cond, null, not_nullable) block.replace_by_position( result, ColumnNullable::create(materialize_column_if_const(arg_else.column), arg_cond.column)); } } else if (cond_const_col) { - if (cond_const_col->get_value()) { + if (cond_const_col->get_value()) { // if(true, null, else) block.get_by_position(result).column = block.get_by_position(result).type->create_column()->clone_resized( input_rows_count); - } else { + } else { // if(false, null, else) block.get_by_position(result).column = make_nullable_column_if_not(arg_else.column); } @@ -353,24 +357,24 @@ class FunctionIf : public IFunction { negated_null_map_data[i] = !null_map_data[i]; } - if (is_column_nullable(*arg_then.column)) { + if (is_column_nullable(*arg_then.column)) { // if(cond, nullable, NULL) auto arg_then_column = arg_then.column; auto result_column = (*std::move(arg_then_column)).mutate(); assert_cast(*result_column) .apply_negated_null_map( assert_cast(*arg_cond.column)); block.replace_by_position(result, std::move(result_column)); - } else { + } else { // if(cond, not_nullable, NULL) block.replace_by_position( result, ColumnNullable::create(materialize_column_if_const(arg_then.column), std::move(negated_null_map))); } } else if (cond_const_col) { - if (cond_const_col->get_value()) { + if (cond_const_col->get_value()) { // if(true, then, NULL) block.get_by_position(result).column = make_nullable_column_if_not(arg_then.column); - } else { + } else { // if(false, then, NULL) block.get_by_position(result).column = block.get_by_position(result).type->create_column()->clone_resized( input_rows_count); @@ -395,7 +399,9 @@ class FunctionIf : public IFunction { auto* then_is_nullable = check_and_get_column(*arg_then.column); auto* else_is_nullable = check_and_get_column(*arg_else.column); - if (!then_is_nullable && !else_is_nullable) return false; + if (!then_is_nullable && !else_is_nullable) { + return false; + } /** Calculate null mask of result and nested column separately. */ @@ -439,6 +445,7 @@ class FunctionIf : public IFunction { } bool execute_for_null_condition(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, const ColumnWithTypeAndName& arg_cond, const ColumnWithTypeAndName& arg_then, const ColumnWithTypeAndName& arg_else, size_t result) { @@ -459,19 +466,13 @@ class FunctionIf : public IFunction { ((ColumnVector&)(nullable->get_nested_column())).get_data().data(); auto rows = nullable->size(); for (size_t i = 0; i < rows; i++) { - nested_bool_data[i] = null_map[i] ? false : nested_bool_data[i]; + nested_bool_data[i] = null_map[i] ? 0 : nested_bool_data[i]; } + auto column_size = block.columns(); + block.insert({nullable->get_nested_column_ptr(), remove_nullable(arg_cond.type), + arg_cond.name}); - Block temporary_block {{nullable->get_nested_column_ptr(), - remove_nullable(arg_cond.type), arg_cond.name}, - arg_then, - arg_else, - block.get_by_position(result)}; - - execute_impl(context, temporary_block, {0, 1, 2}, 3, rows); - - block.get_by_position(result).column = - std::move(temporary_block.get_by_position(3).column); + execute_impl(context, block, {column_size, arguments[1], arguments[2]}, result, rows); return true; } return false; @@ -493,21 +494,9 @@ class FunctionIf : public IFunction { cond_column.column = materialize_column_if_const(cond_column.column); const ColumnWithTypeAndName& arg_cond = block.get_by_position(arguments[0]); - if (auto* then_is_const = check_and_get_column(*arg_then.column)) { - if (check_and_get_column(then_is_const->get_data_column())) { - ColumnWithTypeAndName& then_column = block.get_by_position(arguments[1]); - then_column.column = materialize_column_if_const(then_column.column); - } - } - // if (auto* else_is_const = check_and_get_column(*arg_else.column)) { - // if (check_and_get_column(else_is_const->get_data_column())) { - // ColumnWithTypeAndName& else_column = block.get_by_position(arguments[2]); - // else_column.column = materialize_column_if_const(else_column.column); - // } - // } - Status ret = Status::OK(); - if (execute_for_null_condition(context, block, arg_cond, arg_then, arg_else, result) || + if (execute_for_null_condition(context, block, arguments, arg_cond, arg_then, arg_else, + result) || execute_for_null_then_else(context, block, arg_cond, arg_then, arg_else, result, input_rows_count, ret) || execute_for_nullable_then_else(context, block, arg_cond, arg_then, arg_else, result,