From a25b3e0a7660e5a140f4f52758e52713b15ca091 Mon Sep 17 00:00:00 2001 From: tangruilin Date: Sat, 9 Mar 2024 13:36:29 +0800 Subject: [PATCH] [task#9512] move from_unixtime, now, current_date, current_time to datafusion_functions Signed-off-by: tangruilin --- datafusion/expr/src/built_in_function.rs | 8 -- datafusion/expr/src/expr_fn.rs | 7 -- .../functions/src/datetime/from_unixtime.rs | 79 +++++++++++++++++++ datafusion/functions/src/datetime/mod.rs | 11 +++ .../physical-expr/src/datetime_expressions.rs | 22 ------ datafusion/physical-expr/src/functions.rs | 3 - datafusion/proto/proto/datafusion.proto | 2 +- datafusion/proto/src/generated/pbjson.rs | 3 - datafusion/proto/src/generated/prost.rs | 4 +- .../proto/src/logical_plan/from_proto.rs | 8 +- datafusion/proto/src/logical_plan/to_proto.rs | 1 - 11 files changed, 94 insertions(+), 54 deletions(-) create mode 100644 datafusion/functions/src/datetime/from_unixtime.rs diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs index be10da3669b4..f40f04022281 100644 --- a/datafusion/expr/src/built_in_function.rs +++ b/datafusion/expr/src/built_in_function.rs @@ -242,8 +242,6 @@ pub enum BuiltinScalarFunction { Substr, /// to_hex ToHex, - /// from_unixtime - FromUnixtime, ///now Now, ///current_date @@ -426,7 +424,6 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::Trim => Volatility::Immutable, BuiltinScalarFunction::Upper => Volatility::Immutable, BuiltinScalarFunction::Struct => Volatility::Immutable, - BuiltinScalarFunction::FromUnixtime => Volatility::Immutable, BuiltinScalarFunction::ArrowTypeof => Volatility::Immutable, BuiltinScalarFunction::OverLay => Volatility::Immutable, BuiltinScalarFunction::Levenshtein => Volatility::Immutable, @@ -716,7 +713,6 @@ impl BuiltinScalarFunction { utf8_to_int_type(&input_expr_types[0], "find_in_set") } BuiltinScalarFunction::ToChar => Ok(Utf8), - BuiltinScalarFunction::FromUnixtime => Ok(Timestamp(Second, None)), BuiltinScalarFunction::Now => { Ok(Timestamp(Nanosecond, Some("+00:00".into()))) } @@ -965,9 +961,6 @@ impl BuiltinScalarFunction { ], self.volatility(), ), - BuiltinScalarFunction::FromUnixtime => { - Signature::uniform(1, vec![Int64], self.volatility()) - } BuiltinScalarFunction::Digest => Signature::one_of( vec![ Exact(vec![Utf8, Utf8]), @@ -1371,7 +1364,6 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::DateTrunc => &["date_trunc", "datetrunc"], BuiltinScalarFunction::DatePart => &["date_part", "datepart"], BuiltinScalarFunction::ToChar => &["to_char", "date_format"], - BuiltinScalarFunction::FromUnixtime => &["from_unixtime"], // hashing functions BuiltinScalarFunction::Digest => &["digest"], diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs index ad69208ce9c5..97c5f225fa87 100644 --- a/datafusion/expr/src/expr_fn.rs +++ b/datafusion/expr/src/expr_fn.rs @@ -846,12 +846,6 @@ scalar_expr!( datetime format, "converts a date, time, timestamp or duration to a string based on the provided format" ); -scalar_expr!( - FromUnixtime, - from_unixtime, - unixtime, - "returns the unix time in format" -); scalar_expr!(CurrentDate, current_date, ,"returns current UTC date as a [`DataType::Date32`] value"); scalar_expr!(Now, now, ,"returns current timestamp in nanoseconds, using the same value for all instances of now() in same statement"); scalar_expr!(CurrentTime, current_time, , "returns current UTC time as a [`DataType::Time64`] value"); @@ -1330,7 +1324,6 @@ mod test { test_scalar_expr!(DatePart, date_part, part, date); test_scalar_expr!(DateTrunc, date_trunc, part, date); test_scalar_expr!(DateBin, date_bin, stride, source, origin); - test_scalar_expr!(FromUnixtime, from_unixtime, unixtime); test_scalar_expr!(ArrayAppend, array_append, array, element); test_scalar_expr!(ArraySort, array_sort, array, desc, null_first); diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs new file mode 100644 index 000000000000..c92d9f6570f5 --- /dev/null +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; + +use arrow::datatypes::{DataType, TimeUnit}; + +use datafusion_common::{exec_err, Result}; +use datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; + +#[derive(Debug)] +pub(super) struct FromUnixtimeFunc { + signature: Signature, +} + +impl FromUnixtimeFunc { + pub fn new() -> Self { + Self { + signature: Signature::uniform( + 1, + vec![DataType::Int64], + Volatility::Immutable, + ), + } + } +} + +impl ScalarUDFImpl for FromUnixtimeFunc { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "from_unixtime" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Timestamp(TimeUnit::Second, None)) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result { + if args.len() != 1 { + return exec_err!( + "from_unixtime function requires 1 argument, got {}", + args.len() + ); + } + + match args[0].data_type() { + DataType::Int64 => { + args[0].cast_to(&DataType::Timestamp(TimeUnit::Second, None), None) + } + other => { + exec_err!( + "Unsupported data type {:?} for function from_unixtime", + other + ) + } + } + } +} diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs index 6c9ea1944b1b..cd782d37cb0f 100644 --- a/datafusion/functions/src/datetime/mod.rs +++ b/datafusion/functions/src/datetime/mod.rs @@ -22,12 +22,18 @@ use std::sync::Arc; use datafusion_expr::ScalarUDF; mod common; +mod from_unixtime; mod to_date; mod to_timestamp; mod to_unixtime; // create UDFs make_udf_function!(to_date::ToDateFunc, TO_DATE, to_date); +make_udf_function!( + from_unixtime::FromUnixtimeFunc, + FROM_UNIXTIME, + from_unixtime +); make_udf_function!(to_unixtime::ToUnixtimeFunc, TO_UNIXTIME, to_unixtime); make_udf_function!(to_timestamp::ToTimestampFunc, TO_TIMESTAMP, to_timestamp); make_udf_function!( @@ -107,6 +113,10 @@ pub mod expr_fn { super::to_date().call(args) } + pub fn from_unixtime(args: Vec) -> Expr { + super::from_unixtime().call(args) + } + #[doc = "converts a string and optional formats to a Unixtime"] pub fn to_unixtime(args: Vec) -> Expr { super::to_unixtime().call(args) @@ -142,6 +152,7 @@ pub mod expr_fn { pub fn functions() -> Vec> { vec![ to_date(), + from_unixtime(), to_unixtime(), to_timestamp(), to_timestamp_seconds(), diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs index e2e74a97d5ba..504be78a2afe 100644 --- a/datafusion/physical-expr/src/datetime_expressions.rs +++ b/datafusion/physical-expr/src/datetime_expressions.rs @@ -1144,28 +1144,6 @@ where Ok(b) } -/// from_unixtime() SQL function implementation -pub fn from_unixtime_invoke(args: &[ColumnarValue]) -> Result { - if args.len() != 1 { - return exec_err!( - "from_unixtime function requires 1 argument, got {}", - args.len() - ); - } - - match args[0].data_type() { - DataType::Int64 => { - args[0].cast_to(&DataType::Timestamp(TimeUnit::Second, None), None) - } - other => { - exec_err!( - "Unsupported data type {:?} for function from_unixtime", - other - ) - } - } -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/datafusion/physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs index eebbb1dbea75..5989a4ada10b 100644 --- a/datafusion/physical-expr/src/functions.rs +++ b/datafusion/physical-expr/src/functions.rs @@ -471,9 +471,6 @@ pub fn create_physical_fun( } BuiltinScalarFunction::MakeDate => Arc::new(datetime_expressions::make_date), BuiltinScalarFunction::ToChar => Arc::new(datetime_expressions::to_char), - BuiltinScalarFunction::FromUnixtime => { - Arc::new(datetime_expressions::from_unixtime_invoke) - } BuiltinScalarFunction::InitCap => Arc::new(|args| match args[0].data_type() { DataType::Utf8 => { make_scalar_function_inner(string_expressions::initcap::)(args) diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index c5b20986c324..4ee857e254f6 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -615,7 +615,7 @@ enum ScalarFunction { Coalesce = 63; Power = 64; StructFun = 65; - FromUnixtime = 66; + // 66 was FromUnixtime Atan2 = 67; DateBin = 68; ArrowTypeof = 69; diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index b99e95740622..ef278b6c19d9 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -22378,7 +22378,6 @@ impl serde::Serialize for ScalarFunction { Self::Coalesce => "Coalesce", Self::Power => "Power", Self::StructFun => "StructFun", - Self::FromUnixtime => "FromUnixtime", Self::Atan2 => "Atan2", Self::DateBin => "DateBin", Self::ArrowTypeof => "ArrowTypeof", @@ -22503,7 +22502,6 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Coalesce", "Power", "StructFun", - "FromUnixtime", "Atan2", "DateBin", "ArrowTypeof", @@ -22657,7 +22655,6 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Coalesce" => Ok(ScalarFunction::Coalesce), "Power" => Ok(ScalarFunction::Power), "StructFun" => Ok(ScalarFunction::StructFun), - "FromUnixtime" => Ok(ScalarFunction::FromUnixtime), "Atan2" => Ok(ScalarFunction::Atan2), "DateBin" => Ok(ScalarFunction::DateBin), "ArrowTypeof" => Ok(ScalarFunction::ArrowTypeof), diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index 62b3d39580ae..c76d3e5ca56b 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -2703,7 +2703,7 @@ pub enum ScalarFunction { Coalesce = 63, Power = 64, StructFun = 65, - FromUnixtime = 66, + /// 66 was FromUnixtime Atan2 = 67, DateBin = 68, ArrowTypeof = 69, @@ -2840,7 +2840,6 @@ impl ScalarFunction { ScalarFunction::Coalesce => "Coalesce", ScalarFunction::Power => "Power", ScalarFunction::StructFun => "StructFun", - ScalarFunction::FromUnixtime => "FromUnixtime", ScalarFunction::Atan2 => "Atan2", ScalarFunction::DateBin => "DateBin", ScalarFunction::ArrowTypeof => "ArrowTypeof", @@ -2959,7 +2958,6 @@ impl ScalarFunction { "Coalesce" => Some(Self::Coalesce), "Power" => Some(Self::Power), "StructFun" => Some(Self::StructFun), - "FromUnixtime" => Some(Self::FromUnixtime), "Atan2" => Some(Self::Atan2), "DateBin" => Some(Self::DateBin), "ArrowTypeof" => Some(Self::ArrowTypeof), diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index c26b8acbf1d5..7c2c9f2b64a3 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -56,8 +56,8 @@ use datafusion_expr::{ concat_expr, concat_ws_expr, cos, cosh, cot, current_date, current_time, date_bin, date_part, date_trunc, degrees, digest, ends_with, exp, expr::{self, InList, Sort, WindowFunction}, - factorial, find_in_set, flatten, floor, from_unixtime, gcd, initcap, iszero, lcm, - left, levenshtein, ln, log, log10, log2, + factorial, find_in_set, flatten, floor, gcd, initcap, iszero, lcm, left, levenshtein, + ln, log, log10, log2, logical_plan::{PlanType, StringifiedPlan}, lower, lpad, ltrim, md5, nanvl, now, octet_length, overlay, pi, power, radians, random, repeat, replace, reverse, right, round, rpad, rtrim, sha224, sha256, sha384, @@ -548,7 +548,6 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction { ScalarFunction::Pi => Self::Pi, ScalarFunction::Power => Self::Power, ScalarFunction::StructFun => Self::Struct, - ScalarFunction::FromUnixtime => Self::FromUnixtime, ScalarFunction::Atan2 => Self::Atan2, ScalarFunction::Nanvl => Self::Nanvl, ScalarFunction::Iszero => Self::Iszero, @@ -1773,9 +1772,6 @@ pub fn parse_expr( parse_expr(&args[0], registry, codec)?, parse_expr(&args[1], registry, codec)?, )), - ScalarFunction::FromUnixtime => { - Ok(from_unixtime(parse_expr(&args[0], registry, codec)?)) - } ScalarFunction::Atan2 => Ok(atan2( parse_expr(&args[0], registry, codec)?, parse_expr(&args[1], registry, codec)?, diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index 55c8542d9745..54eb0033ed31 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -1526,7 +1526,6 @@ impl TryFrom<&BuiltinScalarFunction> for protobuf::ScalarFunction { BuiltinScalarFunction::Pi => Self::Pi, BuiltinScalarFunction::Power => Self::Power, BuiltinScalarFunction::Struct => Self::StructFun, - BuiltinScalarFunction::FromUnixtime => Self::FromUnixtime, BuiltinScalarFunction::Atan2 => Self::Atan2, BuiltinScalarFunction::Nanvl => Self::Nanvl, BuiltinScalarFunction::Iszero => Self::Iszero,