Skip to content

Commit

Permalink
Cast int/real as real (#1911) (#1928)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-srebot
ti-srebot authored May 19, 2021
1 parent c947fd6 commit 9da54ca
Show file tree
Hide file tree
Showing 5 changed files with 69 additions and 35 deletions.
17 changes: 17 additions & 0 deletions dbms/src/Debug/dbgFuncCoprocessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ std::unordered_map<String, tipb::ScalarFuncSig> func_name_to_sig({
{"notequals", tipb::ScalarFuncSig::NEInt},
{"like", tipb::ScalarFuncSig::LikeSig},
{"cast_int_int", tipb::ScalarFuncSig::CastIntAsInt},
{"cast_int_real", tipb::ScalarFuncSig::CastIntAsReal},
{"cast_real_int", tipb::ScalarFuncSig::CastRealAsInt},
{"cast_real_real", tipb::ScalarFuncSig::CastRealAsReal},
{"cast_decimal_int", tipb::ScalarFuncSig::CastDecimalAsInt},
{"cast_time_int", tipb::ScalarFuncSig::CastTimeAsInt},
{"cast_string_int", tipb::ScalarFuncSig::CastStringAsInt},
Expand Down Expand Up @@ -765,6 +767,15 @@ void astToPB(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, uint32_t co
}
break;
}
case tipb::ScalarFuncSig::CastIntAsReal:
case tipb::ScalarFuncSig::CastRealAsReal:
{
expr->set_sig(it_sig->second);
auto * ft = expr->mutable_field_type();
ft->set_tp(TiDB::TypeDouble);
ft->set_collate(collator_id);
break;
}
default:
{
expr->set_sig(it_sig->second);
Expand Down Expand Up @@ -1628,6 +1639,12 @@ TiDB::ColumnInfo compileExpr(const DAGSchema & input, ASTPtr ast)
ci.tp = TiDB::TypeDate;
}
break;
case tipb::ScalarFuncSig::CastIntAsReal:
case tipb::ScalarFuncSig::CastRealAsReal:
{
ci.tp = TiDB::TypeDouble;
break;
}
default:
ci.tp = TiDB::TypeLongLong;
ci.flag = TiDB::ColumnFlagUnsigned;
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Flash/Coprocessor/DAGUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -516,7 +516,7 @@ std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
//{tipb::ScalarFuncSig::CastIntAsJson, "cast"},

{tipb::ScalarFuncSig::CastRealAsInt, "tidb_cast"},
//{tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsString, "tidb_cast"}, {tipb::ScalarFuncSig::CastRealAsDecimal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsTime, "tidb_cast"},
//{tipb::ScalarFuncSig::CastRealAsDuration, "cast"},
Expand Down
55 changes: 22 additions & 33 deletions dbms/src/Functions/FunctionsTiDBConversion.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -587,10 +587,16 @@ struct TiDBConvertToFloat
}

template <typename T>
static Float64 toFloat(const DecimalField<T> & value, bool need_truncate, Float64 shift, Float64 max_f, const Context & context)
static std::enable_if_t<std::is_floating_point_v<T> || std::is_integral_v<T>, Float64> toFloat(
const T & value)
{
Float64 float_value = static_cast<Float64>(value);
return produceTargetFloat64(float_value, need_truncate, shift, max_f, context);
return static_cast<Float64>(value);
}

template <typename T>
static Float64 toFloat(const DecimalField<T> & value)
{
return static_cast<Float64>(value);
}

static StringRef getValidFloatPrefix(const StringRef & value)
Expand Down Expand Up @@ -672,15 +678,6 @@ struct TiDBConvertToFloat
vec_null_map_to = &col_null_map_to->getData();
}

bool need_truncate = tp.flen() != -1 && tp.decimal() != -1 && tp.flen() >= tp.decimal();
Float64 shift = 0;
Float64 max_f = 0;
if (need_truncate)
{
shift = std::pow((Float64)10, tp.flen());
max_f = std::pow((Float64)10, tp.flen() - tp.decimal()) - 1.0 / shift;
}

if constexpr (IsDecimal<FromFieldType>)
{
/// cast decimal as real
Expand All @@ -689,7 +686,7 @@ struct TiDBConvertToFloat
for (size_t i = 0; i < size; ++i)
{
auto & field = (*col_from)[i].template safeGet<DecimalField<FromFieldType>>();
vec_to[i] = toFloat(field, need_truncate, shift, max_f, context);
vec_to[i] = toFloat(field);
}
}
else if constexpr (std::is_same_v<FromDataType, DataTypeMyDateTime> || std::is_same_v<FromDataType, DataTypeMyDate>)
Expand All @@ -706,19 +703,17 @@ struct TiDBConvertToFloat
if constexpr (std::is_same_v<DataTypeMyDate, FromDataType>)
{
MyDate date(vec_from[i]);
vec_to[i] = toFloat(date.year * 10000 + date.month * 100 + date.day, need_truncate, shift, max_f, context);
vec_to[i] = toFloat(date.year * 10000 + date.month * 100 + date.day);
}
else
{
MyDateTime date_time(vec_from[i]);
if (type.getFraction() > 0)
vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0,
need_truncate, shift, max_f, context);
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0);
else
vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second,
need_truncate, shift, max_f, context);
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second);
}
}
}
Expand All @@ -731,6 +726,14 @@ struct TiDBConvertToFloat
const ColumnString::Chars_t * chars = &col_from_string->getChars();
const IColumn::Offsets * offsets = &col_from_string->getOffsets();
size_t current_offset = 0;
bool need_truncate = tp.flen() != -1 && tp.decimal() != -1 && tp.flen() >= tp.decimal();
Float64 shift = 0;
Float64 max_f = 0;
if (need_truncate)
{
shift = std::pow((Float64)10, tp.flen());
max_f = std::pow((Float64)10, tp.flen() - tp.decimal()) - 1.0 / shift;
}
for (size_t i = 0; i < size; i++)
{
size_t next_offset = (*offsets)[i];
Expand All @@ -747,7 +750,7 @@ struct TiDBConvertToFloat
= checkAndGetColumn<ColumnVector<FromFieldType>>(block.getByPosition(arguments[0]).column.get());
const typename ColumnVector<FromFieldType>::Container & vec_from = col_from->getData();
for (size_t i = 0; i < size; i++)
vec_to[i] = toFloat(vec_from[i], need_truncate, shift, max_f, context);
vec_to[i] = toFloat(vec_from[i]);
}
else
{
Expand Down Expand Up @@ -1540,20 +1543,6 @@ class FunctionTiDBCast final : public IFunctionBase
block, arguments, result, in_union_, tidb_tp_, context_);
}
};
if (checkDataType<DataTypeFloat32>(to_type.get()))
return [](Block & block, const ColumnNumbers & arguments, const size_t result, bool in_union_, const tipb::FieldType & tidb_tp_,
const Context & context_) {
if (hasUnsignedFlag(tidb_tp_))
{
TiDBConvertToFloat<FromDataType, DataTypeFloat32, return_nullable, true>::execute(
block, arguments, result, in_union_, tidb_tp_, context_);
}
else
{
TiDBConvertToFloat<FromDataType, DataTypeFloat32, return_nullable, false>::execute(
block, arguments, result, in_union_, tidb_tp_, context_);
}
};
/// cast as string
if (checkDataType<DataTypeString>(to_type.get()))
return [](Block & block, const ColumnNumbers & arguments, const size_t result, bool in_union_, const tipb::FieldType & tidb_tp_,
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ class PKSquashingBlockInputStream final : public IBlockInputStream
if (block.rows() > 1 && !isAlreadySorted(block, sort))
stableSortBlock(block, sort);
}
return block;
return std::move(block);
}

private:
Expand Down
28 changes: 28 additions & 0 deletions tests/delta-merge-test/query/expr/cast_as_real.test
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Preparation.
=> DBGInvoke __enable_schema_sync_service('true')

=> DBGInvoke __drop_tidb_table(default, test)
=> drop table if exists default.test

=> DBGInvoke __set_flush_threshold(1000000, 1000000)

# Data.
=> DBGInvoke __mock_tidb_table(default, test, 'col_1 Int8, col_2 UInt8, col_3 Int16, col_4 Nullable(UInt16), col_5 Int32, col_6 UInt32, col_7 Int64, col_8 UInt64, col_9 Nullable(Float32), col_10 Float64')
=> DBGInvoke __refresh_schemas()
=> DBGInvoke __put_region(4, 0, 100, default, test)
=> DBGInvoke __raft_insert_row(default, test, 4, 50, -128, 255, -32768, null, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, null, 1234567.890123)
=> DBGInvoke __raft_insert_row(default, test, 4, 51, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, -12345.6789, 1234567.890123)
=> DBGInvoke __raft_insert_row(default, test, 4, 52, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, null, 1234567.890123)
=> DBGInvoke __raft_insert_row(default, test, 4, 53, -128, 255, -32768, null, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 12345.6789, 1234567.890123)

=> DBGInvoke dag('select cast_int_real(col_1), cast_int_real(col_2), cast_int_real(col_3), cast_int_real(col_4), cast_int_real(col_5),cast_int_real(col_6), cast_int_real(col_7),cast_int_real(col_8), cast_real_real(col_9), cast_real_real(col_10) from default.test', 4,'encode_type:chunk')
┌─cast_int_real(col_1)─┬─cast_int_real(col_2)─┬─cast_int_real(col_3)─┬─cast_int_real(col_4)─┬─cast_int_real(col_5)─┬─cast_int_real(col_6)─┬─cast_int_real(col_7)─┬─cast_int_real(col_8)─┬─cast_real_real(col_9)─┬─cast_real_real(col_10)─┐
│ -128 │ 255 │ -32768 │ \N │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ \N │ 1234567.890123 │
│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ -12345.6787109375 │ 1234567.890123 │
│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ \N │ 1234567.890123 │
│ -128 │ 255 │ -32768 │ \N │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ 12345.6787109375 │ 1234567.890123 │
└──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴───────────────────────┴────────────────────────┘

# Clean up.
=> DBGInvoke __drop_tidb_table(default, test)
=> drop table if exists default.test

0 comments on commit 9da54ca

Please sign in to comment.