Skip to content

Commit

Permalink
Merge pull request #10216 from sundy-li/decimal-tpch
Browse files Browse the repository at this point in the history
  • Loading branch information
@sundy-li
sundy-li authored Mar 1, 2023
2 parents ee12cc7 + 68a0dfc commit ce6d667
Show file tree
Hide file tree
Showing 40 changed files with 419 additions and 152 deletions.
18 changes: 9 additions & 9 deletions benchmark/tpch/prepare_fuse_table.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ echo "CREATE TABLE IF NOT EXISTS part
p_type STRING not null,
p_size INTEGER not null,
p_container STRING not null,
p_retailprice DOUBLE not null,
p_retailprice DECIMAL(15, 2) not null,
p_comment STRING not null
)" | $MYSQL_CLIENT_CONNECT

Expand All @@ -44,7 +44,7 @@ echo "CREATE TABLE IF NOT EXISTS supplier
s_address STRING not null,
s_nationkey INTEGER not null,
s_phone STRING not null,
s_acctbal DOUBLE not null,
s_acctbal DECIMAL(15, 2) not null,
s_comment STRING not null
)" | $MYSQL_CLIENT_CONNECT

Expand All @@ -53,7 +53,7 @@ echo "CREATE TABLE IF NOT EXISTS partsupp
ps_partkey BIGINT not null,
ps_suppkey BIGINT not null,
ps_availqty BIGINT not null,
ps_supplycost DOUBLE not null,
ps_supplycost DECIMAL(15, 2) not null,
ps_comment STRING not null
)" | $MYSQL_CLIENT_CONNECT

Expand All @@ -64,7 +64,7 @@ echo "CREATE TABLE IF NOT EXISTS customer
c_address STRING not null,
c_nationkey INTEGER not null,
c_phone STRING not null,
c_acctbal DOUBLE not null,
c_acctbal DECIMAL(15, 2) not null,
c_mktsegment STRING not null,
c_comment STRING not null
)" | $MYSQL_CLIENT_CONNECT
Expand All @@ -74,7 +74,7 @@ echo "CREATE TABLE IF NOT EXISTS orders
o_orderkey BIGINT not null,
o_custkey BIGINT not null,
o_orderstatus STRING not null,
o_totalprice DOUBLE not null,
o_totalprice DECIMAL(15, 2) not null,
o_orderdate DATE not null,
o_orderpriority STRING not null,
o_clerk STRING not null,
Expand All @@ -88,10 +88,10 @@ echo "CREATE TABLE IF NOT EXISTS lineitem
l_partkey BIGINT not null,
l_suppkey BIGINT not null,
l_linenumber BIGINT not null,
l_quantity DOUBLE not null,
l_extendedprice DOUBLE not null,
l_discount DOUBLE not null,
l_tax DOUBLE not null,
l_quantity DECIMAL(15, 2) not null,
l_extendedprice DECIMAL(15, 2) not null,
l_discount DECIMAL(15, 2) not null,
l_tax DECIMAL(15, 2) not null,
l_returnflag STRING not null,
l_linestatus STRING not null,
l_shipdate DATE not null,
Expand Down
18 changes: 9 additions & 9 deletions benchmark/tpch/prepare_native_table.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ echo "CREATE TABLE IF NOT EXISTS part
p_type STRING not null,
p_size INTEGER not null,
p_container STRING not null,
p_retailprice DOUBLE not null,
p_retailprice DECIMAL(15, 2) not null,
p_comment STRING not null
) storage_format = 'native' compression = 'lz4'" | $MYSQL_CLIENT_CONNECT

Expand All @@ -44,7 +44,7 @@ echo "CREATE TABLE IF NOT EXISTS supplier
s_address STRING not null,
s_nationkey INTEGER not null,
s_phone STRING not null,
s_acctbal DOUBLE not null,
s_acctbal DECIMAL(15, 2) not null,
s_comment STRING not null
) storage_format = 'native' compression = 'lz4'" | $MYSQL_CLIENT_CONNECT

Expand All @@ -53,7 +53,7 @@ echo "CREATE TABLE IF NOT EXISTS partsupp
ps_partkey BIGINT not null,
ps_suppkey BIGINT not null,
ps_availqty BIGINT not null,
ps_supplycost DOUBLE not null,
ps_supplycost DECIMAL(15, 2) not null,
ps_comment STRING not null
) storage_format = 'native' compression = 'lz4'" | $MYSQL_CLIENT_CONNECT

Expand All @@ -64,7 +64,7 @@ echo "CREATE TABLE IF NOT EXISTS customer
c_address STRING not null,
c_nationkey INTEGER not null,
c_phone STRING not null,
c_acctbal DOUBLE not null,
c_acctbal DECIMAL(15, 2) not null,
c_mktsegment STRING not null,
c_comment STRING not null
)" | $MYSQL_CLIENT_CONNECT
Expand All @@ -74,7 +74,7 @@ echo "CREATE TABLE IF NOT EXISTS orders
o_orderkey BIGINT not null,
o_custkey BIGINT not null,
o_orderstatus STRING not null,
o_totalprice DOUBLE not null,
o_totalprice DECIMAL(15, 2) not null,
o_orderdate DATE not null,
o_orderpriority STRING not null,
o_clerk STRING not null,
Expand All @@ -88,10 +88,10 @@ echo "CREATE TABLE IF NOT EXISTS lineitem
l_partkey BIGINT not null,
l_suppkey BIGINT not null,
l_linenumber BIGINT not null,
l_quantity DOUBLE not null,
l_extendedprice DOUBLE not null,
l_discount DOUBLE not null,
l_tax DOUBLE not null,
l_quantity DECIMAL(15, 2) not null,
l_extendedprice DECIMAL(15, 2) not null,
l_discount DECIMAL(15, 2) not null,
l_tax DECIMAL(15, 2) not null,
l_returnflag STRING not null,
l_linestatus STRING not null,
l_shipdate DATE not null,
Expand Down
51 changes: 25 additions & 26 deletions src/query/expression/src/evaluator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
// limitations under the License.

use std::collections::HashMap;
#[cfg(debug_assertions)]
use std::sync::Mutex;

use common_arrow::arrow::bitmap;
use common_exception::ErrorCode;
Expand Down Expand Up @@ -168,30 +166,31 @@ impl<'a> Evaluator<'a> {
}
};

#[cfg(debug_assertions)]
if result.is_err() {
static RECURSING: Mutex<bool> = Mutex::new(false);
if !*RECURSING.lock().unwrap() {
*RECURSING.lock().unwrap() = true;
assert_eq!(
ConstantFolder::fold_with_domain(
expr,
self.input_columns
.domains()
.into_iter()
.enumerate()
.collect(),
self.func_ctx,
self.fn_registry
)
.1,
None,
"domain calculation should not return any domain for expressions that are possible to fail"
);
*RECURSING.lock().unwrap() = false;
}
}

// We can't call this in debug mode, because it will cause infinite recursion.
// Eg: select 3.2::Decimal(10, 2)::Int32;
// #[cfg(debug_assertions)]
// if result.is_err() {
// static RECURSING: Mutex<bool> = Mutex::new(false);
// if !*RECURSING.lock().unwrap() {
// *RECURSING.lock().unwrap() = true;
// assert_eq!(
// ConstantFolder::fold_with_domain(
// expr,
// self.input_columns
// .domains()
// .into_iter()
// .enumerate()
// .collect(),
// self.func_ctx,
// self.fn_registry
// )
// .1,
// None,
// "domain calculation should not return any domain for expressions that are possible to fail"
// );
// *RECURSING.lock().unwrap() = false;
// }
// }
result
}

Expand Down
14 changes: 14 additions & 0 deletions src/query/expression/src/property.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ use crate::types::NumberType;
use crate::types::StringType;
use crate::types::TimestampType;
use crate::types::ValueType;
use crate::with_decimal_type;
use crate::with_number_type;
use crate::Scalar;

Expand Down Expand Up @@ -193,6 +194,19 @@ impl Domain {
_ => unreachable!("unable to merge {this:?} with {other:?}"),
})
}
(Domain::Decimal(this), Domain::Decimal(other)) => {
with_decimal_type!(|TYPE| match (this, other) {
(DecimalDomain::TYPE(x, size), DecimalDomain::TYPE(y, _)) =>
Domain::Decimal(DecimalDomain::TYPE(
SimpleDomain {
min: x.min.min(y.min),
max: x.max.max(y.max),
},
*size
),),
_ => unreachable!("unable to merge {this:?} with {other:?}"),
})
}
(Domain::Boolean(this), Domain::Boolean(other)) => Domain::Boolean(BooleanDomain {
has_false: this.has_false || other.has_false,
has_true: this.has_true || other.has_true,
Expand Down
25 changes: 20 additions & 5 deletions src/query/expression/src/type_check.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,11 @@ use crate::expression::Literal;
use crate::expression::RawExpr;
use crate::function::FunctionRegistry;
use crate::function::FunctionSignature;
use crate::types::decimal::DecimalSize;
use crate::types::number::NumberDataType;
use crate::types::number::NumberScalar;
use crate::types::DataType;
use crate::types::DecimalDataType;
use crate::AutoCastRules;
use crate::ColumnIndex;
use crate::Scalar;
Expand Down Expand Up @@ -516,11 +518,24 @@ pub fn common_super_type(
.collect::<Option<Vec<_>>>()?;
Some(DataType::Tuple(tys))
}
// todo!("decimal")
// (
// DataType::Number(_) | DataType::Decimal(_),
// DataType::Number(_) | DataType::Decimal(_),
// ) => DataType::Decimal(?),
(DataType::Number(_), DataType::Decimal(ty))
| (DataType::Decimal(ty), DataType::Number(_)) => {
let max_precision = ty.max_precision();
let scale = ty.scale();

DecimalDataType::from_size(DecimalSize {
precision: max_precision,
scale,
})
.ok()
.map(DataType::Decimal)
}

(DataType::Decimal(a), DataType::Decimal(b)) => {
let ty = DecimalDataType::binary_result_type(&a, &b, false, false, true).ok();
ty.map(DataType::Decimal)
}

(ty1, ty2) => {
let ty1_can_cast_to = auto_cast_rules
.iter()
Expand Down
75 changes: 74 additions & 1 deletion src/query/functions/src/scalars/decimal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ macro_rules! register_decimal_compare_op {
DataType::Decimal(return_type.clone()),
DataType::Decimal(return_type.clone()),
],
return_type: DataType::Decimal(return_type.clone()),
return_type: DataType::Boolean,
property: FunctionProperty::default(),
},
calc_domain: Box::new(|_args_domain| FunctionDomain::Full),
Expand Down Expand Up @@ -396,6 +396,29 @@ pub fn register(registry: &mut FunctionRegistry) {
}),
}))
});

// decimal to float
registry.register_function_factory("to_float64", |_params, args_type| {
if args_type.len() != 1 {
return None;
}

let arg_type = args_type[0].clone();
if !arg_type.is_decimal() {
return None;
}

Some(Arc::new(Function {
signature: FunctionSignature {
name: "to_float64".to_string(),
args_type: vec![arg_type.clone()],
return_type: Float64Type::data_type(),
property: FunctionProperty::default(),
},
calc_domain: Box::new(|_args_domain| FunctionDomain::Full),
eval: Box::new(move |args, tx| decimal_to_float64(args, arg_type.clone(), tx)),
}))
});
}

fn convert_to_decimal(
Expand Down Expand Up @@ -662,3 +685,53 @@ fn decimal_to_decimal(
Value::Column(Column::Decimal(result))
}
}

fn decimal_to_float64(
args: &[ValueRef<AnyType>],
from_type: DataType,
_ctx: &mut EvalContext,
) -> Value<AnyType> {
let arg = &args[0];

let mut is_scalar = false;
let column = match arg {
ValueRef::Column(column) => column.clone(),
ValueRef::Scalar(s) => {
is_scalar = true;
let builder = ColumnBuilder::repeat(s, 1, &from_type);
builder.build()
}
};

let from_type = from_type.as_decimal().unwrap();

let result = match from_type {
DecimalDataType::Decimal128(_) => {
let (buffer, from_size) = i128::try_downcast_column(&column).unwrap();

let div = 10_f64.powi(from_size.scale as i32);

let values: Buffer<F64> = buffer.iter().map(|x| (*x as f64 / div).into()).collect();
Float64Type::upcast_column(values)
}

DecimalDataType::Decimal256(_) => {
let (buffer, from_size) = i256::try_downcast_column(&column).unwrap();

let div = 10_f64.powi(from_size.scale as i32);

let values: Buffer<F64> = buffer
.iter()
.map(|x| (f64::from(*x) / div).into())
.collect();
Float64Type::upcast_column(values)
}
};

if is_scalar {
let scalar = result.index(0).unwrap();
Value::Scalar(scalar.to_owned())
} else {
Value::Column(result)
}
}
1 change: 1 addition & 0 deletions src/query/functions/tests/it/scalars/testdata/function_list.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3252,6 +3252,7 @@ regexp_like
regexp_replace
regexp_substr
to_decimal
to_float64
tuple

Function aliases (alias to origin):
Expand Down
11 changes: 10 additions & 1 deletion tests/sqllogictests/suites/base/11_data_type/11_0006_data_type_decimal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,16 @@ drop table t;
statement error 1049
create table t (a Decimal(0, 0));

query I
query IIII
select 1::Decimal(17, 3), 2::Decimal(19, 4), 3::Decimal(20, 5), cast(10 as Decimal(7, 2));
----
1.000 2.0000 3.00000 10.00

query IIII
select 1::Decimal(17, 3)::Float64, 2::Decimal(19, 4)::Float64, 3::Decimal(20, 5)::Float64, cast(10 as Decimal(7, 2))::Float64;
----
1.0 2.0 3.0 10.0

## parser overflow
statement error 1005
select 2::Decimal(2000, 3);
Expand Down Expand Up @@ -193,6 +198,10 @@ select a > b, a < b, a = b, a <= b, a >= b from (select 3::Decimal(13,2) a , 2.9
----
1 0 0 0 1

query T
select typeof(a = b) from (select 3::Decimal(13,2) a , 2.9 b);
----
BOOLEAN

## insert

Expand Down
Loading

1 comment on commit ce6d667

@vercel
Copy link

@vercel vercel bot commented on ce6d667 Mar 1, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

databend – ./

databend-databend.vercel.app
databend-git-main-databend.vercel.app
databend.rs
databend.vercel.app

Please sign in to comment.