From f16c643af2510fedcf95094c0ae91d347b3605f0 Mon Sep 17 00:00:00 2001 From: fzhedu Date: Fri, 14 May 2021 18:00:17 +0800 Subject: [PATCH 1/2] cast int/real as real --- dbms/src/Debug/dbgFuncCoprocessor.cpp | 17 ++++++ dbms/src/Flash/Coprocessor/DAGUtils.cpp | 2 +- dbms/src/Functions/FunctionsTiDBConversion.h | 55 ++++++++----------- .../DeltaMerge/PKSquashingBlockInputStream.h | 2 +- 4 files changed, 41 insertions(+), 35 deletions(-) diff --git a/dbms/src/Debug/dbgFuncCoprocessor.cpp b/dbms/src/Debug/dbgFuncCoprocessor.cpp index 84f220ae2f4..d5276a372c8 100644 --- a/dbms/src/Debug/dbgFuncCoprocessor.cpp +++ b/dbms/src/Debug/dbgFuncCoprocessor.cpp @@ -102,7 +102,9 @@ std::unordered_map func_name_to_sig({ {"notequals", tipb::ScalarFuncSig::NEInt}, {"like", tipb::ScalarFuncSig::LikeSig}, {"cast_int_int", tipb::ScalarFuncSig::CastIntAsInt}, + {"cast_int_real", tipb::ScalarFuncSig::CastIntAsReal}, {"cast_real_int", tipb::ScalarFuncSig::CastRealAsInt}, + {"cast_real_real", tipb::ScalarFuncSig::CastRealAsReal}, {"cast_decimal_int", tipb::ScalarFuncSig::CastDecimalAsInt}, {"cast_time_int", tipb::ScalarFuncSig::CastTimeAsInt}, {"cast_string_int", tipb::ScalarFuncSig::CastStringAsInt}, @@ -765,6 +767,15 @@ void astToPB(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, uint32_t co } break; } + case tipb::ScalarFuncSig::CastIntAsReal: + case tipb::ScalarFuncSig::CastRealAsReal: + { + expr->set_sig(it_sig->second); + auto * ft = expr->mutable_field_type(); + ft->set_tp(TiDB::TypeDouble); + ft->set_collate(collator_id); + break; + } default: { expr->set_sig(it_sig->second); @@ -1628,6 +1639,12 @@ TiDB::ColumnInfo compileExpr(const DAGSchema & input, ASTPtr ast) ci.tp = TiDB::TypeDate; } break; + case tipb::ScalarFuncSig::CastIntAsReal: + case tipb::ScalarFuncSig::CastRealAsReal: + { + ci.tp = TiDB::TypeDouble; + break; + } default: ci.tp = TiDB::TypeLongLong; ci.flag = TiDB::ColumnFlagUnsigned; diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 76fd8fab1be..4f12f42806e 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -516,7 +516,7 @@ std::unordered_map scalar_func_map({ //{tipb::ScalarFuncSig::CastIntAsJson, "cast"}, {tipb::ScalarFuncSig::CastRealAsInt, "tidb_cast"}, - //{tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"}, + {tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"}, {tipb::ScalarFuncSig::CastRealAsString, "tidb_cast"}, {tipb::ScalarFuncSig::CastRealAsDecimal, "tidb_cast"}, {tipb::ScalarFuncSig::CastRealAsTime, "tidb_cast"}, //{tipb::ScalarFuncSig::CastRealAsDuration, "cast"}, diff --git a/dbms/src/Functions/FunctionsTiDBConversion.h b/dbms/src/Functions/FunctionsTiDBConversion.h index 74e3122a7ec..81611d1004d 100644 --- a/dbms/src/Functions/FunctionsTiDBConversion.h +++ b/dbms/src/Functions/FunctionsTiDBConversion.h @@ -588,10 +588,16 @@ struct TiDBConvertToFloat } template - static Float64 toFloat(const DecimalField & value, bool need_truncate, Float64 shift, Float64 max_f, const Context & context) + static std::enable_if_t || std::is_integral_v, Float64> toFloat( + const T & value) { - Float64 float_value = static_cast(value); - return produceTargetFloat64(float_value, need_truncate, shift, max_f, context); + return static_cast(value); + } + + template + static Float64 toFloat(const DecimalField & value) + { + return static_cast(value); } static StringRef getValidFloatPrefix(const StringRef & value) @@ -673,15 +679,6 @@ struct TiDBConvertToFloat vec_null_map_to = &col_null_map_to->getData(); } - bool need_truncate = tp.flen() != -1 && tp.decimal() != -1 && tp.flen() >= tp.decimal(); - Float64 shift = 0; - Float64 max_f = 0; - if (need_truncate) - { - shift = std::pow((Float64)10, tp.flen()); - max_f = std::pow((Float64)10, tp.flen() - tp.decimal()) - 1.0 / shift; - } - if constexpr (IsDecimal) { /// cast decimal as real @@ -690,7 +687,7 @@ struct TiDBConvertToFloat for (size_t i = 0; i < size; ++i) { auto & field = (*col_from)[i].template safeGet>(); - vec_to[i] = toFloat(field, need_truncate, shift, max_f, context); + vec_to[i] = toFloat(field); } } else if constexpr (std::is_same_v || std::is_same_v) @@ -707,19 +704,17 @@ struct TiDBConvertToFloat if constexpr (std::is_same_v) { MyDate date(vec_from[i]); - vec_to[i] = toFloat(date.year * 10000 + date.month * 100 + date.day, need_truncate, shift, max_f, context); + vec_to[i] = toFloat(date.year * 10000 + date.month * 100 + date.day); } else { MyDateTime date_time(vec_from[i]); if (type.getFraction() > 0) vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000 - + date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0, - need_truncate, shift, max_f, context); + + date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0); else vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000 - + date_time.hour * 1000 + date_time.minute * 100 + date_time.second, - need_truncate, shift, max_f, context); + + date_time.hour * 1000 + date_time.minute * 100 + date_time.second); } } } @@ -732,6 +727,14 @@ struct TiDBConvertToFloat const ColumnString::Chars_t * chars = &col_from_string->getChars(); const IColumn::Offsets * offsets = &col_from_string->getOffsets(); size_t current_offset = 0; + bool need_truncate = tp.flen() != -1 && tp.decimal() != -1 && tp.flen() >= tp.decimal(); + Float64 shift = 0; + Float64 max_f = 0; + if (need_truncate) + { + shift = std::pow((Float64)10, tp.flen()); + max_f = std::pow((Float64)10, tp.flen() - tp.decimal()) - 1.0 / shift; + } for (size_t i = 0; i < size; i++) { size_t next_offset = (*offsets)[i]; @@ -748,7 +751,7 @@ struct TiDBConvertToFloat = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get()); const typename ColumnVector::Container & vec_from = col_from->getData(); for (size_t i = 0; i < size; i++) - vec_to[i] = toFloat(vec_from[i], need_truncate, shift, max_f, context); + vec_to[i] = toFloat(vec_from[i]); } else { @@ -1652,20 +1655,6 @@ class FunctionTiDBCast final : public IFunctionBase block, arguments, result, in_union_, tidb_tp_, context_); } }; - if (checkDataType(to_type.get())) - return [](Block & block, const ColumnNumbers & arguments, const size_t result, bool in_union_, const tipb::FieldType & tidb_tp_, - const Context & context_) { - if (hasUnsignedFlag(tidb_tp_)) - { - TiDBConvertToFloat::execute( - block, arguments, result, in_union_, tidb_tp_, context_); - } - else - { - TiDBConvertToFloat::execute( - block, arguments, result, in_union_, tidb_tp_, context_); - } - }; /// cast as string if (checkDataType(to_type.get())) return [](Block & block, const ColumnNumbers & arguments, const size_t result, bool in_union_, const tipb::FieldType & tidb_tp_, diff --git a/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h b/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h index 31f698d18a7..5e8964e4577 100644 --- a/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h +++ b/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h @@ -149,7 +149,7 @@ class PKSquashingBlockInputStream final : public IBlockInputStream if (block.rows() > 1 && !isAlreadySorted(block, sort)) stableSortBlock(block, sort); } - return block; + return std::move(block); } private: From cfe56cd96dee0dfa0c386f64fd2f5f04f7542950 Mon Sep 17 00:00:00 2001 From: fzhedu Date: Fri, 14 May 2021 18:00:27 +0800 Subject: [PATCH 2/2] cast int/real as real test --- .../query/expr/cast_as_real.test | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/delta-merge-test/query/expr/cast_as_real.test diff --git a/tests/delta-merge-test/query/expr/cast_as_real.test b/tests/delta-merge-test/query/expr/cast_as_real.test new file mode 100644 index 00000000000..84b3da9d1ce --- /dev/null +++ b/tests/delta-merge-test/query/expr/cast_as_real.test @@ -0,0 +1,28 @@ +# Preparation. +=> DBGInvoke __enable_schema_sync_service('true') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +# Data. +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 Int8, col_2 UInt8, col_3 Int16, col_4 Nullable(UInt16), col_5 Int32, col_6 UInt32, col_7 Int64, col_8 UInt64, col_9 Nullable(Float32), col_10 Float64') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, -128, 255, -32768, null, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, null, 1234567.890123) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, -12345.6789, 1234567.890123) +=> DBGInvoke __raft_insert_row(default, test, 4, 52, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, null, 1234567.890123) +=> DBGInvoke __raft_insert_row(default, test, 4, 53, -128, 255, -32768, null, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 12345.6789, 1234567.890123) + +=> DBGInvoke dag('select cast_int_real(col_1), cast_int_real(col_2), cast_int_real(col_3), cast_int_real(col_4), cast_int_real(col_5),cast_int_real(col_6), cast_int_real(col_7),cast_int_real(col_8), cast_real_real(col_9), cast_real_real(col_10) from default.test', 4,'encode_type:chunk') +┌─cast_int_real(col_1)─┬─cast_int_real(col_2)─┬─cast_int_real(col_3)─┬─cast_int_real(col_4)─┬─cast_int_real(col_5)─┬─cast_int_real(col_6)─┬─cast_int_real(col_7)─┬─cast_int_real(col_8)─┬─cast_real_real(col_9)─┬─cast_real_real(col_10)─┐ +│ -128 │ 255 │ -32768 │ \N │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ \N │ 1234567.890123 │ +│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ -12345.6787109375 │ 1234567.890123 │ +│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ \N │ 1234567.890123 │ +│ -128 │ 255 │ -32768 │ \N │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ 12345.6787109375 │ 1234567.890123 │ +└──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴───────────────────────┴────────────────────────┘ + +# Clean up. +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test