Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cast int/real as real #1911

Merged
merged 3 commits into from
May 18, 2021
Merged

Cast int/real as real #1911

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f16c643
cast int/real as real
fzhedu May 14, 2021
cfe56cd
cast int/real as real test
fzhedu May 14, 2021
dfcac24
Merge branch 'master' into castXXAsReal
ti-srebot May 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions dbms/src/Debug/dbgFuncCoprocessor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ std::unordered_map<String, tipb::ScalarFuncSig> func_name_to_sig({
{"notequals", tipb::ScalarFuncSig::NEInt},
{"like", tipb::ScalarFuncSig::LikeSig},
{"cast_int_int", tipb::ScalarFuncSig::CastIntAsInt},
{"cast_int_real", tipb::ScalarFuncSig::CastIntAsReal},
{"cast_real_int", tipb::ScalarFuncSig::CastRealAsInt},
{"cast_real_real", tipb::ScalarFuncSig::CastRealAsReal},
{"cast_decimal_int", tipb::ScalarFuncSig::CastDecimalAsInt},
{"cast_time_int", tipb::ScalarFuncSig::CastTimeAsInt},
{"cast_string_int", tipb::ScalarFuncSig::CastStringAsInt},
Expand Down Expand Up @@ -765,6 +767,15 @@ void astToPB(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, uint32_t co
}
break;
}
case tipb::ScalarFuncSig::CastIntAsReal:
case tipb::ScalarFuncSig::CastRealAsReal:
{
expr->set_sig(it_sig->second);
auto * ft = expr->mutable_field_type();
ft->set_tp(TiDB::TypeDouble);
ft->set_collate(collator_id);
break;
}
default:
{
expr->set_sig(it_sig->second);
Expand Down Expand Up @@ -1628,6 +1639,12 @@ TiDB::ColumnInfo compileExpr(const DAGSchema & input, ASTPtr ast)
ci.tp = TiDB::TypeDate;
}
break;
case tipb::ScalarFuncSig::CastIntAsReal:
case tipb::ScalarFuncSig::CastRealAsReal:
{
ci.tp = TiDB::TypeDouble;
break;
}
default:
ci.tp = TiDB::TypeLongLong;
ci.flag = TiDB::ColumnFlagUnsigned;
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Flash/Coprocessor/DAGUtils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,7 +516,7 @@ std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
//{tipb::ScalarFuncSig::CastIntAsJson, "cast"},

{tipb::ScalarFuncSig::CastRealAsInt, "tidb_cast"},
//{tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsReal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsString, "tidb_cast"}, {tipb::ScalarFuncSig::CastRealAsDecimal, "tidb_cast"},
{tipb::ScalarFuncSig::CastRealAsTime, "tidb_cast"},
//{tipb::ScalarFuncSig::CastRealAsDuration, "cast"},
Expand Down
55 changes: 22 additions & 33 deletions dbms/src/Functions/FunctionsTiDBConversion.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,10 +588,16 @@ struct TiDBConvertToFloat
}

template <typename T>
static Float64 toFloat(const DecimalField<T> & value, bool need_truncate, Float64 shift, Float64 max_f, const Context & context)
static std::enable_if_t<std::is_floating_point_v<T> || std::is_integral_v<T>, Float64> toFloat(
const T & value)
{
Float64 float_value = static_cast<Float64>(value);
return produceTargetFloat64(float_value, need_truncate, shift, max_f, context);
return static_cast<Float64>(value);
}

template <typename T>
static Float64 toFloat(const DecimalField<T> & value)
{
return static_cast<Float64>(value);
}

static StringRef getValidFloatPrefix(const StringRef & value)
Expand Down Expand Up @@ -673,15 +679,6 @@ struct TiDBConvertToFloat
vec_null_map_to = &col_null_map_to->getData();
}

bool need_truncate = tp.flen() != -1 && tp.decimal() != -1 && tp.flen() >= tp.decimal();
Float64 shift = 0;
Float64 max_f = 0;
if (need_truncate)
{
shift = std::pow((Float64)10, tp.flen());
max_f = std::pow((Float64)10, tp.flen() - tp.decimal()) - 1.0 / shift;
}

if constexpr (IsDecimal<FromFieldType>)
{
/// cast decimal as real
Expand All @@ -690,7 +687,7 @@ struct TiDBConvertToFloat
for (size_t i = 0; i < size; ++i)
{
auto & field = (*col_from)[i].template safeGet<DecimalField<FromFieldType>>();
vec_to[i] = toFloat(field, need_truncate, shift, max_f, context);
vec_to[i] = toFloat(field);
}
}
else if constexpr (std::is_same_v<FromDataType, DataTypeMyDateTime> || std::is_same_v<FromDataType, DataTypeMyDate>)
Expand All @@ -707,19 +704,17 @@ struct TiDBConvertToFloat
if constexpr (std::is_same_v<DataTypeMyDate, FromDataType>)
{
MyDate date(vec_from[i]);
vec_to[i] = toFloat(date.year * 10000 + date.month * 100 + date.day, need_truncate, shift, max_f, context);
vec_to[i] = toFloat(date.year * 10000 + date.month * 100 + date.day);
}
else
{
MyDateTime date_time(vec_from[i]);
if (type.getFraction() > 0)
vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0,
need_truncate, shift, max_f, context);
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second + date_time.micro_second / 1000000.0);
else
vec_to[i] = toFloat(date_time.year * 10000000000ULL + date_time.month * 100000000ULL + date_time.day * 100000
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second,
need_truncate, shift, max_f, context);
+ date_time.hour * 1000 + date_time.minute * 100 + date_time.second);
}
}
}
Expand All @@ -732,6 +727,14 @@ struct TiDBConvertToFloat
const ColumnString::Chars_t * chars = &col_from_string->getChars();
const IColumn::Offsets * offsets = &col_from_string->getOffsets();
size_t current_offset = 0;
bool need_truncate = tp.flen() != -1 && tp.decimal() != -1 && tp.flen() >= tp.decimal();
Float64 shift = 0;
Float64 max_f = 0;
if (need_truncate)
{
shift = std::pow((Float64)10, tp.flen());
max_f = std::pow((Float64)10, tp.flen() - tp.decimal()) - 1.0 / shift;
}
for (size_t i = 0; i < size; i++)
{
size_t next_offset = (*offsets)[i];
Expand All @@ -748,7 +751,7 @@ struct TiDBConvertToFloat
= checkAndGetColumn<ColumnVector<FromFieldType>>(block.getByPosition(arguments[0]).column.get());
const typename ColumnVector<FromFieldType>::Container & vec_from = col_from->getData();
for (size_t i = 0; i < size; i++)
vec_to[i] = toFloat(vec_from[i], need_truncate, shift, max_f, context);
vec_to[i] = toFloat(vec_from[i]);
}
else
{
Expand Down Expand Up @@ -1652,20 +1655,6 @@ class FunctionTiDBCast final : public IFunctionBase
block, arguments, result, in_union_, tidb_tp_, context_);
}
};
if (checkDataType<DataTypeFloat32>(to_type.get()))
return [](Block & block, const ColumnNumbers & arguments, const size_t result, bool in_union_, const tipb::FieldType & tidb_tp_,
const Context & context_) {
if (hasUnsignedFlag(tidb_tp_))
{
TiDBConvertToFloat<FromDataType, DataTypeFloat32, return_nullable, true>::execute(
block, arguments, result, in_union_, tidb_tp_, context_);
}
else
{
TiDBConvertToFloat<FromDataType, DataTypeFloat32, return_nullable, false>::execute(
block, arguments, result, in_union_, tidb_tp_, context_);
}
};
/// cast as string
if (checkDataType<DataTypeString>(to_type.get()))
return [](Block & block, const ColumnNumbers & arguments, const size_t result, bool in_union_, const tipb::FieldType & tidb_tp_,
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ class PKSquashingBlockInputStream final : public IBlockInputStream
if (block.rows() > 1 && !isAlreadySorted(block, sort))
stableSortBlock(block, sort);
}
return block;
return std::move(block);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we don't need to std::move here, since it should be handled by NRVO.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it cause compilation error on my mac env.

}

private:
Expand Down
28 changes: 28 additions & 0 deletions tests/delta-merge-test/query/expr/cast_as_real.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Preparation.
=> DBGInvoke __enable_schema_sync_service('true')

=> DBGInvoke __drop_tidb_table(default, test)
=> drop table if exists default.test

=> DBGInvoke __set_flush_threshold(1000000, 1000000)

# Data.
=> DBGInvoke __mock_tidb_table(default, test, 'col_1 Int8, col_2 UInt8, col_3 Int16, col_4 Nullable(UInt16), col_5 Int32, col_6 UInt32, col_7 Int64, col_8 UInt64, col_9 Nullable(Float32), col_10 Float64')
=> DBGInvoke __refresh_schemas()
=> DBGInvoke __put_region(4, 0, 100, default, test)
=> DBGInvoke __raft_insert_row(default, test, 4, 50, -128, 255, -32768, null, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, null, 1234567.890123)
=> DBGInvoke __raft_insert_row(default, test, 4, 51, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, -12345.6789, 1234567.890123)
=> DBGInvoke __raft_insert_row(default, test, 4, 52, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, null, 1234567.890123)
=> DBGInvoke __raft_insert_row(default, test, 4, 53, -128, 255, -32768, null, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 12345.6789, 1234567.890123)

=> DBGInvoke dag('select cast_int_real(col_1), cast_int_real(col_2), cast_int_real(col_3), cast_int_real(col_4), cast_int_real(col_5),cast_int_real(col_6), cast_int_real(col_7),cast_int_real(col_8), cast_real_real(col_9), cast_real_real(col_10) from default.test', 4,'encode_type:chunk')
┌─cast_int_real(col_1)─┬─cast_int_real(col_2)─┬─cast_int_real(col_3)─┬─cast_int_real(col_4)─┬─cast_int_real(col_5)─┬─cast_int_real(col_6)─┬─cast_int_real(col_7)─┬─cast_int_real(col_8)─┬─cast_real_real(col_9)─┬─cast_real_real(col_10)─┐
│ -128 │ 255 │ -32768 │ \N │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ \N │ 1234567.890123 │
│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ -12345.6787109375 │ 1234567.890123 │
│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ \N │ 1234567.890123 │
│ -128 │ 255 │ -32768 │ \N │ -2147483648 │ 4294967295 │ -9223372036854776000 │ 18446744073709552000 │ 12345.6787109375 │ 1234567.890123 │
└──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴───────────────────────┴────────────────────────┘

# Clean up.
=> DBGInvoke __drop_tidb_table(default, test)
=> drop table if exists default.test