From 745cb2983a9ab1157cc9afe234c16ff17df95523 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Feb 2023 16:56:52 +0800 Subject: [PATCH 1/7] chore(query): introduce TzLUT util --- Cargo.lock | 2 + src/query/expression/Cargo.toml | 2 + src/query/expression/src/function.rs | 14 +- src/query/expression/src/types/date.rs | 1 + src/query/expression/src/types/timestamp.rs | 1 + src/query/expression/src/types/variant.rs | 12 +- src/query/expression/src/utils/date_helper.rs | 284 +++++++++++++----- src/query/expression/src/utils/display.rs | 6 +- src/query/functions/benches/bench.rs | 5 +- src/query/functions/src/scalars/datetime.rs | 42 +-- src/query/functions/src/scalars/variant.rs | 9 +- .../functions/tests/it/aggregates/mod.rs | 5 +- src/query/service/src/sessions/query_ctx.rs | 6 +- 13 files changed, 264 insertions(+), 125 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fe81ca717b55..89593b6d9fb3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1389,6 +1389,7 @@ dependencies = [ "common-exception", "common-io", "common-jsonb", + "dashmap", "educe", "enum-as-inner", "enum_dispatch", @@ -1401,6 +1402,7 @@ dependencies = [ "match-template", "micromarshal", "num-traits", + "once_cell", "ordered-float 3.4.0", "parking_lot 0.12.1", "pretty_assertions", diff --git a/src/query/expression/Cargo.toml b/src/query/expression/Cargo.toml index fa4fd96df621d..7d4fbea10bc16 100755 --- a/src/query/expression/Cargo.toml +++ b/src/query/expression/Cargo.toml @@ -23,6 +23,7 @@ chrono = { workspace = true } chrono-tz = { workspace = true } comfy-table = "6" common-jsonb = { path = "../../common/jsonb" } +dashmap = "5.4" educe = "0.4" enum-as-inner = "0.5" enum_dispatch = "0.3.8" @@ -34,6 +35,7 @@ lexical-core = "0.8.5" match-template = "0.0.1" micromarshal = "0.2.1" num-traits = "0.2.15" +once_cell = "1.15.0" ordered-float = { workspace = true, features = ["serde", "rand"] } parking_lot = "0.12.1" primitive-types = "0.12.0" diff --git a/src/query/expression/src/function.rs b/src/query/expression/src/function.rs index b1371f1a2432c..134e16bcf6cca 100755 --- a/src/query/expression/src/function.rs +++ b/src/query/expression/src/function.rs @@ -16,7 +16,6 @@ use std::collections::HashMap; use std::ops::BitAnd; use std::sync::Arc; -use chrono_tz::Tz; use common_arrow::arrow::bitmap::Bitmap; use common_arrow::arrow::bitmap::MutableBitmap; use common_exception::ErrorCode; @@ -26,6 +25,7 @@ use itertools::Itertools; use serde::Deserialize; use serde::Serialize; +use crate::date_helper::TzLUT; use crate::property::Domain; use crate::property::FunctionProperty; use crate::types::nullable::NullableColumn; @@ -49,22 +49,16 @@ pub struct FunctionSignature { pub type AutoCastSignature = Vec<(DataType, DataType)>; -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Default)] pub struct FunctionContext { - pub tz: Tz, -} - -impl Default for FunctionContext { - fn default() -> Self { - Self { tz: Tz::UTC } - } + pub tz: TzLUT, } #[derive(Clone)] pub struct EvalContext<'a> { pub generics: &'a GenericMap, pub num_rows: usize, - pub tz: Tz, + pub tz: TzLUT, /// Validity bitmap of outer nullable column. This is an optimization /// to avoid recording errors on the NULL value which has a coresponding diff --git a/src/query/expression/src/types/date.rs b/src/query/expression/src/types/date.rs index f351b9790404f..148f86fc233a7 100644 --- a/src/query/expression/src/types/date.rs +++ b/src/query/expression/src/types/date.rs @@ -25,6 +25,7 @@ use num_traits::AsPrimitive; use super::number::SimpleDomain; use crate::date_helper::DateConverter; + use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; diff --git a/src/query/expression/src/types/timestamp.rs b/src/query/expression/src/types/timestamp.rs index 792d9244bf943..408090014ccb8 100644 --- a/src/query/expression/src/types/timestamp.rs +++ b/src/query/expression/src/types/timestamp.rs @@ -23,6 +23,7 @@ use common_io::cursor_ext::BufferReadDateTimeExt; use common_io::cursor_ext::ReadBytesExt; use super::number::SimpleDomain; + use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; diff --git a/src/query/expression/src/types/variant.rs b/src/query/expression/src/types/variant.rs index a39bcec33d8fd..250173a68c3cd 100644 --- a/src/query/expression/src/types/variant.rs +++ b/src/query/expression/src/types/variant.rs @@ -14,11 +14,12 @@ use std::ops::Range; -use chrono_tz::Tz; + use super::date::date_to_string; use super::number::NumberScalar; use super::timestamp::timestamp_to_string; +use crate::date_helper::TzLUT; use crate::property::Domain; use crate::types::string::StringColumn; use crate::types::string::StringColumnBuilder; @@ -170,7 +171,8 @@ impl ArgType for VariantType { } } -pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: Tz, buf: &mut Vec) { +pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: TzLUT, buf: &mut Vec) { + let inner_tz = tz.tz; let value = match scalar { ScalarRef::Null => common_jsonb::Value::Null, ScalarRef::EmptyArray => common_jsonb::Value::Array(vec![]), @@ -189,8 +191,8 @@ pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: Tz, buf: &mut Vec) { ScalarRef::Decimal(_) => todo!("decimal"), ScalarRef::Boolean(b) => common_jsonb::Value::Bool(b), ScalarRef::String(s) => common_jsonb::Value::String(String::from_utf8_lossy(s)), - ScalarRef::Timestamp(ts) => timestamp_to_string(ts, tz).to_string().into(), - ScalarRef::Date(d) => date_to_string(d, tz).to_string().into(), + ScalarRef::Timestamp(ts) => timestamp_to_string(ts, inner_tz).to_string().into(), + ScalarRef::Date(d) => date_to_string(d, inner_tz).to_string().into(), ScalarRef::Array(col) => { let items = cast_scalars_to_variants(col.iter(), tz); common_jsonb::build_array(items.iter(), buf).expect("failed to build jsonb array"); @@ -218,7 +220,7 @@ pub fn cast_scalar_to_variant(scalar: ScalarRef, tz: Tz, buf: &mut Vec) { pub fn cast_scalars_to_variants( scalars: impl IntoIterator, - tz: Tz, + tz: TzLUT, ) -> StringColumn { let iter = scalars.into_iter(); let mut builder = StringColumnBuilder::with_capacity(iter.size_hint().0, 0); diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index dc1ff14d39ced..c90a5ad5cf118 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -16,17 +16,205 @@ use chrono::Date; use chrono::DateTime; use chrono::Datelike; use chrono::Duration; +use chrono::LocalResult; use chrono::NaiveDate; use chrono::NaiveDateTime; +use chrono::NaiveTime; use chrono::Offset; use chrono::TimeZone; use chrono::Timelike; use chrono::Utc; use chrono_tz::Tz; +use common_exception::ErrorCode; +use common_exception::Result; use num_traits::AsPrimitive; +use once_cell::sync::Lazy; use crate::types::date::check_date; use crate::types::timestamp::check_timestamp; +use crate::types::timestamp::MICROS_IN_A_SEC; + +#[derive(Debug, Clone, Copy)] +pub struct TzLUT { + pub tz: Tz, + pub offset_round_hour: bool, + pub offset_round_minute: bool, +} + +impl Default for TzLUT { + fn default() -> Self { + Self { + tz: Tz::UTC, + offset_round_hour: true, + offset_round_minute: true, + } + } +} + +static TZ_FACTORY: Lazy = Lazy::new(|| { + let factory = TzFactory { + luts: dashmap::DashMap::new(), + }; + let _ = factory.get(Tz::UTC); + let _ = factory.get(Tz::Asia__Shanghai); + let _ = factory.get(Tz::Asia__Tokyo); + let _ = factory.get(Tz::America__New_York); + let _ = factory.get(Tz::Europe__London); + factory +}); + +pub struct TzFactory { + pub luts: dashmap::DashMap, +} + +impl TzFactory { + pub fn instance() -> &'static TzFactory { + &TZ_FACTORY + } + + pub fn get_by_name(&self, tz_name: &str) -> Result { + if let Some(lut) = self.luts.get(tz_name) { + return Ok(*lut.value()); + } + + let tz = tz_name.parse::().map_err(|_| { + ErrorCode::InvalidTimezone("Timezone has been checked and should be valid") + })?; + let lut = TzLUT::new(tz); + self.luts.insert(tz_name.to_string(), lut); + Ok(lut) + } + + pub fn get(&self, tz: Tz) -> TzLUT { + let tz_name = tz.name(); + if let Some(lut) = self.luts.get(tz_name) { + return *lut.value(); + } + let lut = TzLUT::new(tz); + self.luts.insert(tz_name.to_string(), lut); + lut + } +} + +impl TzLUT { + // it's very heavy to initial a TzLUT + fn new(tz: Tz) -> Self { + static DATE_LUT_MIN_YEAR: i32 = 1925; + static DATE_LUT_MAX_YEAR: i32 = 2283; + + let mut offset_round_hour = true; + let mut offset_round_minute = true; + + let date = NaiveDate::from_ymd(DATE_LUT_MIN_YEAR, 1, 1); + let mut days = date.num_days_from_ce(); + + loop { + let time = NaiveDateTime::new( + NaiveDate::from_num_days_from_ce(days), + NaiveTime::from_hms(0, 0, 0), + ); + if time.year() > DATE_LUT_MAX_YEAR { + break; + } + + days += 1; + + match tz.offset_from_local_datetime(&time) { + LocalResult::Single(offset) => { + let offset = offset.fix(); + if offset_round_hour && offset.local_minus_utc() % 3600 != 0 { + offset_round_hour = false; + } + if offset_round_minute && offset.local_minus_utc() % 60 != 0 { + offset_round_minute = false; + } + } + _ => { + continue; + } + } + } + Self { + tz, + offset_round_hour, + offset_round_minute, + } + } + + #[allow(dead_code)] + #[inline] + fn to_start_of_seconds(&self, ts: i64, seconds: i64) -> i64 { + if seconds == 1 { + return ts; + } + if seconds % 60 == 0 { + return self.to_start_of_minutes(ts, seconds); + } + self.round_down(ts, seconds) + } + + #[inline] + fn to_start_of_minutes(&self, ts: i64, seconds_div: i64) -> i64 { + if self.offset_round_minute { + return if ts > 0 { + ts / seconds_div * seconds_div + } else { + (ts + 1 - seconds_div) / seconds_div * seconds_div + }; + } + let datetime = self.tz.timestamp(ts, 0); + + let fix = datetime.offset().fix().local_minus_utc() as i64; + fix + (ts - fix) / seconds_div * seconds_div + } + + #[inline] + fn round_down(&self, ts: i64, seconds_div: i64) -> i64 { + if self.offset_round_hour { + return if ts > 0 { + ts / seconds_div * seconds_div + } else { + (ts + 1 - seconds_div) / seconds_div * seconds_div + }; + } + let datetime = self.tz.timestamp(ts, 0); + let fix = datetime.offset().fix().local_minus_utc() as i64; + fix + (ts - fix) / seconds_div * seconds_div + } + + #[inline] + pub fn round_timestamp_micros(&self, micros: i64, round: Round) -> i64 { + self.round_timestamp(micros / MICROS_IN_A_SEC, round) * MICROS_IN_A_SEC + } + + #[inline] + pub fn round_timestamp(&self, ts: i64, round: Round) -> i64 { + match round { + Round::Second => return ts, + Round::Minute => self.to_start_of_minutes(ts, 60), + Round::FiveMinutes => self.to_start_of_minutes(ts, 15 * 60), + Round::TenMinutes => self.to_start_of_minutes(ts, 10 * 60), + Round::FifteenMinutes => self.to_start_of_minutes(ts, 15 * 60), + Round::TimeSlot => self.to_start_of_minutes(ts, 30 * 60), + Round::Hour => self.round_down(ts, 3600), + Round::Day => self.round_down(ts, 3600 * 24), + } + } + + pub fn to_minute(&self, ts: i64) -> u8 { + if ts >= 0 && self.offset_round_hour { + (ts % 60) as u8 + } else { + let datetime = self.tz.timestamp(ts, 0); + datetime.minute() as u8 + } + } + + pub fn to_hour(&self, ts: i64) -> u8 { + let datetime = self.tz.timestamp(ts, 0); + datetime.hour() as u8 + } +} pub trait DateConverter { fn to_date(&self, tz: Tz) -> Date; @@ -45,10 +233,7 @@ where T: AsPrimitive fn to_timestamp(&self, tz: Tz) -> DateTime { // Can't use `tz.timestamp_nanos(self.as_() * 1000)` directly, is may cause multiply with overflow. let micros = self.as_(); - let (mut secs, mut nanos) = ( - micros / ROUND_MICRO_SECOND, - (micros % ROUND_MICRO_SECOND) * 1_000, - ); + let (mut secs, mut nanos) = (micros / MICROS_IN_A_SEC, (micros % MICROS_IN_A_SEC) * 1_000); if nanos < 0 { secs -= 1; nanos += 1_000_000_000; @@ -115,10 +300,10 @@ macro_rules! impl_interval_year_month { impl $name { pub fn eval_date( date: i32, - tz: Tz, + tz: TzLUT, delta: impl AsPrimitive, ) -> Result { - let date = date.to_date(tz); + let date = date.to_date(tz.tz); let new_date = $op(date.year(), date.month(), date.day(), delta.as_())?; check_date( new_date @@ -129,10 +314,10 @@ macro_rules! impl_interval_year_month { pub fn eval_timestamp( ts: i64, - tz: Tz, + tz: TzLUT, delta: impl AsPrimitive, ) -> Result { - let ts = ts.to_timestamp(tz); + let ts = ts.to_timestamp(tz.tz); let new_ts = $op(ts.year(), ts.month(), ts.day(), delta.as_())?; check_timestamp(NaiveDateTime::new(new_ts, ts.time()).timestamp_micros()) } @@ -151,7 +336,7 @@ impl AddDaysImpl { } pub fn eval_timestamp(date: i64, delta: impl AsPrimitive) -> Result { - check_timestamp(date.wrapping_add(delta.as_() * 24 * 3600 * ROUND_MICRO_SECOND)) + check_timestamp(date.wrapping_add(delta.as_() * 24 * 3600 * MICROS_IN_A_SEC)) } } @@ -160,8 +345,8 @@ pub struct AddTimesImpl; impl AddTimesImpl { pub fn eval_date(date: i32, delta: impl AsPrimitive, factor: i64) -> Result { check_date( - (date as i64 * 3600 * 24 * ROUND_MICRO_SECOND) - .wrapping_add(delta.as_() * factor * ROUND_MICRO_SECOND), + (date as i64 * 3600 * 24 * MICROS_IN_A_SEC) + .wrapping_add(delta.as_() * factor * MICROS_IN_A_SEC), ) } @@ -170,7 +355,7 @@ impl AddTimesImpl { delta: impl AsPrimitive, factor: i64, ) -> Result { - check_timestamp(ts.wrapping_add(delta.as_() * factor * ROUND_MICRO_SECOND)) + check_timestamp(ts.wrapping_add(delta.as_() * factor * MICROS_IN_A_SEC)) } } @@ -189,13 +374,13 @@ pub trait ToNumber { pub struct ToNumberImpl; impl ToNumberImpl { - pub fn eval_timestamp, R>(ts: i64, tz: Tz) -> R { - let dt = ts.to_timestamp(tz); + pub fn eval_timestamp, R>(ts: i64, tz: TzLUT) -> R { + let dt = ts.to_timestamp(tz.tz); T::to_number(&dt) } - pub fn eval_date, R>(date: i32, tz: Tz) -> R { - let dt = date.to_date(tz).and_hms(0, 0, 0); + pub fn eval_date, R>(date: i32, tz: TzLUT) -> R { + let dt = date.to_date(tz.tz).and_hms(0, 0, 0); T::to_number(&dt) } } @@ -265,24 +450,6 @@ impl ToNumber for ToDayOfWeek { } } -impl ToNumber for ToHour { - fn to_number(dt: &DateTime) -> u8 { - dt.hour() as u8 - } -} - -impl ToNumber for ToMinute { - fn to_number(dt: &DateTime) -> u8 { - dt.minute() as u8 - } -} - -impl ToNumber for ToSecond { - fn to_number(dt: &DateTime) -> u8 { - dt.second() as u8 - } -} - #[derive(Clone, Copy)] pub enum Round { Second, @@ -295,57 +462,16 @@ pub enum Round { Day, } -const ROUND_MICRO_SECOND: i64 = 1_000_000; -pub fn round_timestamp(ts: i64, tz: Tz, round: Round) -> i64 { - let second = ts / ROUND_MICRO_SECOND; - let dt = tz.timestamp(second, 0_u32); - if matches!(round, Round::Minute) && dt.offset().fix().local_minus_utc() % 60 == 0 { - return (second / 60) * 60 * ROUND_MICRO_SECOND; - } - - let res = match round { - Round::Second => dt, - Round::Minute => { - return (second - dt.second() as i64) * ROUND_MICRO_SECOND; - } - Round::FiveMinutes => { - return (second - (dt.second() as i64 + (dt.minute() % 5) as i64 * 60)) - * ROUND_MICRO_SECOND; - } - Round::TenMinutes => { - return (second - (dt.second() as i64 + (dt.minute() % 10) as i64 * 60)) - * ROUND_MICRO_SECOND; - } - Round::FifteenMinutes => { - return (second - (dt.second() as i64 + (dt.minute() % 15) as i64 * 60)) - * ROUND_MICRO_SECOND; - } - Round::TimeSlot => tz.ymd(dt.year(), dt.month(), dt.day()).and_hms_micro( - dt.hour(), - dt.minute() / 30 * 30, - 0, - 0, - ), - Round::Hour => tz - .ymd(dt.year(), dt.month(), dt.day()) - .and_hms_micro(dt.hour(), 0, 0, 0), - Round::Day => tz - .ymd(dt.year(), dt.month(), dt.day()) - .and_hms_micro(0, 0, 0, 0), - }; - res.timestamp_micros() -} - pub struct DateRounder; impl DateRounder { - pub fn eval_timestamp>(ts: i64, tz: Tz) -> i32 { - let dt = ts.to_timestamp(tz); + pub fn eval_timestamp>(ts: i64, tz: TzLUT) -> i32 { + let dt = ts.to_timestamp(tz.tz); T::to_number(&dt) } - pub fn eval_date>(date: i32, tz: Tz) -> i32 { - let dt = date.to_date(tz).and_hms(0, 0, 0); + pub fn eval_date>(date: i32, tz: TzLUT) -> i32 { + let dt = date.to_date(tz.tz).and_hms(0, 0, 0); T::to_number(&dt) } } diff --git a/src/query/expression/src/utils/display.rs b/src/query/expression/src/utils/display.rs index 2a9dc525410f9..59a87591a40fe 100755 --- a/src/query/expression/src/utils/display.rs +++ b/src/query/expression/src/utils/display.rs @@ -17,6 +17,7 @@ use std::fmt::Display; use std::fmt::Formatter; use std::fmt::Write; +use chrono_tz::Tz; use comfy_table::Cell; use comfy_table::Table; use ethnum::i256; @@ -26,6 +27,7 @@ use rust_decimal::Decimal; use rust_decimal::RoundingStrategy; use crate::block::DataBlock; + use crate::expression::Expr; use crate::expression::Literal; use crate::expression::RawExpr; @@ -181,8 +183,8 @@ impl<'a> Display for ScalarRef<'a> { Ok(()) } }, - ScalarRef::Timestamp(t) => write!(f, "{}", timestamp_to_string(*t, chrono_tz::Tz::UTC)), - ScalarRef::Date(d) => write!(f, "{}", date_to_string(*d as i64, chrono_tz::Tz::UTC)), + ScalarRef::Timestamp(t) => write!(f, "{}", timestamp_to_string(*t, Tz::UTC)), + ScalarRef::Date(d) => write!(f, "{}", date_to_string(*d as i64, Tz::UTC)), ScalarRef::Array(col) => write!(f, "[{}]", col.iter().join(", ")), ScalarRef::Tuple(fields) => { write!(f, "(")?; diff --git a/src/query/functions/benches/bench.rs b/src/query/functions/benches/bench.rs index a660dfb744737..e26ef59cd705e 100644 --- a/src/query/functions/benches/bench.rs +++ b/src/query/functions/benches/bench.rs @@ -18,6 +18,7 @@ extern crate criterion; #[path = "../tests/it/scalars/parser.rs"] mod parser; +use common_expression::date_helper::TzLUT; use common_expression::type_check; use common_expression::DataBlock; use common_expression::Evaluator; @@ -41,7 +42,9 @@ fn bench(c: &mut Criterion) { b.iter(|| type_check::check(&raw_expr, &BUILTIN_FUNCTIONS)) }); - let func_ctx = FunctionContext { tz: chrono_tz::UTC }; + let func_ctx = FunctionContext { + tz: TzLUT::default(), + }; let expr = type_check::check(&raw_expr, &BUILTIN_FUNCTIONS).unwrap(); let block = DataBlock::new(vec![], 1); let evaluator = Evaluator::new(&block, func_ctx, &BUILTIN_FUNCTIONS); diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 0b63e02c563a7..0a15a9d229756 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -141,7 +141,7 @@ fn register_string_to_timestamp(registry: &mut FunctionRegistry) { ctx: &mut EvalContext, ) -> Value { vectorize_with_builder_1_arg::(|val, output, ctx| { - match string_to_timestamp(val, ctx.tz) { + match string_to_timestamp(val, ctx.tz.tz) { Some(ts) => output.push(ts.timestamp_micros()), None => { ctx.set_error(output.len(), "unable to parse string to type `TIMESTAMP`"); @@ -249,7 +249,7 @@ fn register_string_to_date(registry: &mut FunctionRegistry) { fn eval_string_to_date(val: ValueRef, ctx: &mut EvalContext) -> Value { vectorize_with_builder_1_arg::( - |val, output, ctx| match string_to_date(val, ctx.tz) { + |val, output, ctx| match string_to_date(val, ctx.tz.tz) { Some(d) => output.push(d.num_days_from_ce() - EPOCH_DAYS_FROM_CE), None => { ctx.set_error(output.len(), "unable to parse string to type `DATE`"); @@ -343,7 +343,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - write!(output.data, "{}", timestamp_to_string(val, ctx.tz)).unwrap(); + write!(output.data, "{}", timestamp_to_string(val, ctx.tz.tz)).unwrap(); output.commit_row(); }), ); @@ -353,7 +353,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_with_builder_1_arg::(|val, output, ctx| { - write!(output.data, "{}", date_to_string(val, ctx.tz)).unwrap(); + write!(output.data, "{}", date_to_string(val, ctx.tz.tz)).unwrap(); output.commit_row(); }), ); @@ -372,7 +372,12 @@ fn register_to_string(registry: &mut FunctionRegistry) { }, vectorize_with_builder_1_arg::>( |val, output, ctx| { - write!(output.builder.data, "{}", timestamp_to_string(val, ctx.tz)).unwrap(); + write!( + output.builder.data, + "{}", + timestamp_to_string(val, ctx.tz.tz) + ) + .unwrap(); output.builder.commit_row(); output.validity.push(true); }, @@ -392,7 +397,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { }) }, vectorize_with_builder_1_arg::>(|val, output, ctx| { - write!(output.builder.data, "{}", date_to_string(val, ctx.tz)).unwrap(); + write!(output.builder.data, "{}", date_to_string(val, ctx.tz.tz)).unwrap(); output.builder.commit_row(); output.validity.push(true); }), @@ -891,12 +896,13 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { ToNumberImpl::eval_timestamp::(val, ctx.tz) }), ); + registry.register_passthrough_nullable_1_arg::( "to_hour", FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.tz) + ctx.tz.to_hour(val / MICROS_IN_A_SEC) }), ); registry.register_passthrough_nullable_1_arg::( @@ -904,16 +910,14 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.tz) + ctx.tz.to_minute(val / MICROS_IN_A_SEC) }), ); registry.register_passthrough_nullable_1_arg::( "to_second", FunctionProperty::default(), |_| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - ToNumberImpl::eval_timestamp::(val, ctx.tz) - }), + vectorize_1_arg::(|val, _ctx| (val / MICROS_IN_A_SEC % 60) as u8), ); } @@ -1084,7 +1088,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::Second) + ctx.tz.round_timestamp_micros(val, Round::Second) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1092,7 +1096,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::Minute) + ctx.tz.round_timestamp_micros(val, Round::Minute) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1100,7 +1104,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::FiveMinutes) + ctx.tz.round_timestamp_micros(val, Round::FiveMinutes) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1108,7 +1112,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::TenMinutes) + ctx.tz.round_timestamp_micros(val, Round::TenMinutes) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1116,7 +1120,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::FifteenMinutes) + ctx.tz.round_timestamp_micros(val, Round::FifteenMinutes) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1124,7 +1128,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::Hour) + ctx.tz.round_timestamp_micros(val, Round::Hour) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1132,7 +1136,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::Day) + ctx.tz.round_timestamp_micros(val, Round::Day) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1140,7 +1144,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - round_timestamp(val, ctx.tz, Round::TimeSlot) + ctx.tz.round_timestamp_micros(val, Round::TimeSlot) }), ); diff --git a/src/query/functions/src/scalars/variant.rs b/src/query/functions/src/scalars/variant.rs index 6e9a993a57cf5..d538c0cf6a4c9 100644 --- a/src/query/functions/src/scalars/variant.rs +++ b/src/query/functions/src/scalars/variant.rs @@ -466,7 +466,7 @@ pub fn register(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|val, output, ctx| { - match as_str(val).and_then(|val| string_to_date(val.as_bytes(), ctx.tz)) { + match as_str(val).and_then(|val| string_to_date(val.as_bytes(), ctx.tz.tz)) { Some(d) => output.push(d.num_days_from_ce() - EPOCH_DAYS_FROM_CE), None => { ctx.set_error(output.len(), "unable to cast to type `DATE`"); @@ -481,7 +481,8 @@ pub fn register(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_with_builder_1_arg::>(|val, output, ctx| { - match as_str(val).and_then(|str_value| string_to_date(str_value.as_bytes(), ctx.tz)) { + match as_str(val).and_then(|str_value| string_to_date(str_value.as_bytes(), ctx.tz.tz)) + { Some(date) => output.push(date.num_days_from_ce() - EPOCH_DAYS_FROM_CE), None => output.push_null(), } @@ -494,7 +495,7 @@ pub fn register(registry: &mut FunctionRegistry) { |_| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::( |val, output, ctx| match as_str(val) - .and_then(|val| string_to_timestamp(val.as_bytes(), ctx.tz)) + .and_then(|val| string_to_timestamp(val.as_bytes(), ctx.tz.tz)) { Some(ts) => output.push(ts.timestamp_micros()), None => { @@ -512,7 +513,7 @@ pub fn register(registry: &mut FunctionRegistry) { vectorize_with_builder_1_arg::>( |val, output, ctx| match as_str(val) { Some(str_val) => { - let timestamp = string_to_timestamp(str_val.as_bytes(), ctx.tz) + let timestamp = string_to_timestamp(str_val.as_bytes(), ctx.tz.tz) .map(|ts| ts.timestamp_micros()); match timestamp { Some(timestamp) => output.push(timestamp), diff --git a/src/query/functions/tests/it/aggregates/mod.rs b/src/query/functions/tests/it/aggregates/mod.rs index 584cbb4e896f7..848d84850f0f3 100644 --- a/src/query/functions/tests/it/aggregates/mod.rs +++ b/src/query/functions/tests/it/aggregates/mod.rs @@ -19,6 +19,7 @@ use std::io::Write; use bumpalo::Bump; use comfy_table::Table; use common_exception::Result; +use common_expression::date_helper::TzLUT; use common_expression::type_check; use common_expression::types::number::NumberScalar; use common_expression::types::AnyType; @@ -148,7 +149,9 @@ pub fn run_scalar_expr( block: &DataBlock, ) -> Result<(Value, DataType)> { let expr = type_check::check(raw_expr, &BUILTIN_FUNCTIONS)?; - let func_ctx = FunctionContext { tz: chrono_tz::UTC }; + let func_ctx = FunctionContext { + tz: TzLUT::default(), + }; let evaluator = Evaluator::new(block, func_ctx, &BUILTIN_FUNCTIONS); let result = evaluator.run(&expr)?; Ok((result, expr.data_type().clone())) diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 016a12e2bd97d..31cb42137e612 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -23,7 +23,6 @@ use std::sync::Arc; use std::sync::Weak; use std::time::SystemTime; -use chrono_tz::Tz; use common_base::base::tokio::task::JoinHandle; use common_base::base::Progress; use common_base::base::ProgressValues; @@ -37,6 +36,7 @@ use common_catalog::table_context::StageAttachment; use common_config::DATABEND_COMMIT_VERSION; use common_exception::ErrorCode; use common_exception::Result; +use common_expression::date_helper::TzFactory; use common_expression::DataBlock; use common_expression::FunctionContext; use common_expression::Scalar; @@ -335,9 +335,7 @@ impl TableContext for QueryContext { fn get_function_context(&self) -> Result { let tz = self.get_settings().get_timezone()?; - let tz = tz.parse::().map_err(|_| { - ErrorCode::InvalidTimezone("Timezone has been checked and should be valid") - })?; + let tz = TzFactory::instance().get_by_name(&tz)?; Ok(FunctionContext { tz }) } From 1d78e02852bea0c5e917156a841224aea44b1089 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Feb 2023 17:00:54 +0800 Subject: [PATCH 2/7] chore(query): introduce TzLUT util --- src/query/expression/src/types/date.rs | 1 - src/query/expression/src/types/timestamp.rs | 1 - src/query/expression/src/types/variant.rs | 2 -- src/query/expression/src/utils/date_helper.rs | 18 +++++++++--------- src/query/expression/src/utils/display.rs | 1 - 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/query/expression/src/types/date.rs b/src/query/expression/src/types/date.rs index 148f86fc233a7..f351b9790404f 100644 --- a/src/query/expression/src/types/date.rs +++ b/src/query/expression/src/types/date.rs @@ -25,7 +25,6 @@ use num_traits::AsPrimitive; use super::number::SimpleDomain; use crate::date_helper::DateConverter; - use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; diff --git a/src/query/expression/src/types/timestamp.rs b/src/query/expression/src/types/timestamp.rs index 408090014ccb8..792d9244bf943 100644 --- a/src/query/expression/src/types/timestamp.rs +++ b/src/query/expression/src/types/timestamp.rs @@ -23,7 +23,6 @@ use common_io::cursor_ext::BufferReadDateTimeExt; use common_io::cursor_ext::ReadBytesExt; use super::number::SimpleDomain; - use crate::property::Domain; use crate::types::ArgType; use crate::types::DataType; diff --git a/src/query/expression/src/types/variant.rs b/src/query/expression/src/types/variant.rs index 250173a68c3cd..049678b623239 100644 --- a/src/query/expression/src/types/variant.rs +++ b/src/query/expression/src/types/variant.rs @@ -14,8 +14,6 @@ use std::ops::Range; - - use super::date::date_to_string; use super::number::NumberScalar; use super::timestamp::timestamp_to_string; diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index c90a5ad5cf118..b5347ae732f12 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -143,18 +143,18 @@ impl TzLUT { #[allow(dead_code)] #[inline] - fn to_start_of_seconds(&self, ts: i64, seconds: i64) -> i64 { + fn start_of_second(&self, ts: i64, seconds: i64) -> i64 { if seconds == 1 { return ts; } if seconds % 60 == 0 { - return self.to_start_of_minutes(ts, seconds); + return self.start_of_minutes(ts, seconds); } self.round_down(ts, seconds) } #[inline] - fn to_start_of_minutes(&self, ts: i64, seconds_div: i64) -> i64 { + fn start_of_minutes(&self, ts: i64, seconds_div: i64) -> i64 { if self.offset_round_minute { return if ts > 0 { ts / seconds_div * seconds_div @@ -190,12 +190,12 @@ impl TzLUT { #[inline] pub fn round_timestamp(&self, ts: i64, round: Round) -> i64 { match round { - Round::Second => return ts, - Round::Minute => self.to_start_of_minutes(ts, 60), - Round::FiveMinutes => self.to_start_of_minutes(ts, 15 * 60), - Round::TenMinutes => self.to_start_of_minutes(ts, 10 * 60), - Round::FifteenMinutes => self.to_start_of_minutes(ts, 15 * 60), - Round::TimeSlot => self.to_start_of_minutes(ts, 30 * 60), + Round::Second => ts, + Round::Minute => self.start_of_minutes(ts, 60), + Round::FiveMinutes => self.start_of_minutes(ts, 15 * 60), + Round::TenMinutes => self.start_of_minutes(ts, 10 * 60), + Round::FifteenMinutes => self.start_of_minutes(ts, 15 * 60), + Round::TimeSlot => self.start_of_minutes(ts, 30 * 60), Round::Hour => self.round_down(ts, 3600), Round::Day => self.round_down(ts, 3600 * 24), } diff --git a/src/query/expression/src/utils/display.rs b/src/query/expression/src/utils/display.rs index 59a87591a40fe..041ba176a59de 100755 --- a/src/query/expression/src/utils/display.rs +++ b/src/query/expression/src/utils/display.rs @@ -27,7 +27,6 @@ use rust_decimal::Decimal; use rust_decimal::RoundingStrategy; use crate::block::DataBlock; - use crate::expression::Expr; use crate::expression::Literal; use crate::expression::RawExpr; From 5134d3160bbe537bca678e783378d7b6c1dedbd1 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Feb 2023 17:07:27 +0800 Subject: [PATCH 3/7] chore(query): introduce TzLUT util --- src/query/expression/src/utils/date_helper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index b5347ae732f12..a996c3d491c9a 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -203,7 +203,7 @@ impl TzLUT { pub fn to_minute(&self, ts: i64) -> u8 { if ts >= 0 && self.offset_round_hour { - (ts % 60) as u8 + ((ts / 60) % 60) as u8 } else { let datetime = self.tz.timestamp(ts, 0); datetime.minute() as u8 From b1d42d51b8d31b6ae92b933498fb7d1d6bfa06aa Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Feb 2023 17:36:01 +0800 Subject: [PATCH 4/7] update --- src/query/expression/src/utils/date_helper.rs | 2 +- src/query/functions/src/scalars/datetime.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index a996c3d491c9a..b5646bb06e552 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -192,7 +192,7 @@ impl TzLUT { match round { Round::Second => ts, Round::Minute => self.start_of_minutes(ts, 60), - Round::FiveMinutes => self.start_of_minutes(ts, 15 * 60), + Round::FiveMinutes => self.start_of_minutes(ts, 5 * 60), Round::TenMinutes => self.start_of_minutes(ts, 10 * 60), Round::FifteenMinutes => self.start_of_minutes(ts, 15 * 60), Round::TimeSlot => self.start_of_minutes(ts, 30 * 60), diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 0a15a9d229756..3e6af2712c95b 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -917,7 +917,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_second", FunctionProperty::default(), |_| FunctionDomain::Full, - vectorize_1_arg::(|val, _ctx| (val / MICROS_IN_A_SEC % 60) as u8), + vectorize_1_arg::(|val, _ctx| (val / MICROS_IN_A_SEC) as u8), ); } From 1bcce9edfdb13136ec9b7c1804ae4c823f46ee2b Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Feb 2023 17:57:04 +0800 Subject: [PATCH 5/7] update --- src/query/functions/src/scalars/datetime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 3e6af2712c95b..0a15a9d229756 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -917,7 +917,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_second", FunctionProperty::default(), |_| FunctionDomain::Full, - vectorize_1_arg::(|val, _ctx| (val / MICROS_IN_A_SEC) as u8), + vectorize_1_arg::(|val, _ctx| (val / MICROS_IN_A_SEC % 60) as u8), ); } From afcd136b1175b522db2b2aa37ff741c344c9a4db Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Feb 2023 21:45:23 +0800 Subject: [PATCH 6/7] update --- src/query/expression/src/utils/date_helper.rs | 103 ++++++++++-------- src/query/functions/src/scalars/datetime.rs | 26 ++--- 2 files changed, 70 insertions(+), 59 deletions(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index b5646bb06e552..f3303b1b14509 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -37,7 +37,9 @@ use crate::types::timestamp::MICROS_IN_A_SEC; #[derive(Debug, Clone, Copy)] pub struct TzLUT { pub tz: Tz, + // whether the timezone offset is round hour, offset % 3600 == 0 pub offset_round_hour: bool, + // whether the timezone offset is round minute, offset % 60 == 0 pub offset_round_minute: bool, } @@ -143,75 +145,88 @@ impl TzLUT { #[allow(dead_code)] #[inline] - fn start_of_second(&self, ts: i64, seconds: i64) -> i64 { + fn start_of_second(&self, us: i64, seconds: i64) -> i64 { if seconds == 1 { - return ts; + return us / MICROS_IN_A_SEC * MICROS_IN_A_SEC; } if seconds % 60 == 0 { - return self.start_of_minutes(ts, seconds); + return self.start_of_minutes(us, seconds / 60); } - self.round_down(ts, seconds) + self.round_down(us, seconds) } #[inline] - fn start_of_minutes(&self, ts: i64, seconds_div: i64) -> i64 { + fn start_of_minutes(&self, us: i64, minus: i64) -> i64 { + let us_div = minus * 60 * MICROS_IN_A_SEC; if self.offset_round_minute { - return if ts > 0 { - ts / seconds_div * seconds_div + return if us > 0 { + us / us_div * us_div } else { - (ts + 1 - seconds_div) / seconds_div * seconds_div + (us + MICROS_IN_A_SEC - us_div) / us_div * us_div }; } - let datetime = self.tz.timestamp(ts, 0); - + let datetime = self.to_datetime_from_us(us); let fix = datetime.offset().fix().local_minus_utc() as i64; - fix + (ts - fix) / seconds_div * seconds_div + fix + (us - fix * MICROS_IN_A_SEC) / us_div * us_div } #[inline] - fn round_down(&self, ts: i64, seconds_div: i64) -> i64 { + fn round_down(&self, us: i64, seconds: i64) -> i64 { + let us_div = seconds * MICROS_IN_A_SEC; if self.offset_round_hour { - return if ts > 0 { - ts / seconds_div * seconds_div + return if us > 0 { + us / us_div * us_div } else { - (ts + 1 - seconds_div) / seconds_div * seconds_div + (us + MICROS_IN_A_SEC - us_div) / us_div * us_div }; } - let datetime = self.tz.timestamp(ts, 0); + let datetime = self.to_datetime_from_us(us); let fix = datetime.offset().fix().local_minus_utc() as i64; - fix + (ts - fix) / seconds_div * seconds_div + fix + (us - fix * MICROS_IN_A_SEC) / us_div * us_div } #[inline] - pub fn round_timestamp_micros(&self, micros: i64, round: Round) -> i64 { - self.round_timestamp(micros / MICROS_IN_A_SEC, round) * MICROS_IN_A_SEC + pub fn round_us(&self, us: i64, round: Round) -> i64 { + match round { + Round::Second => self.start_of_second(us, 1), + Round::Minute => self.start_of_minutes(us, 1), + Round::FiveMinutes => self.start_of_minutes(us, 5), + Round::TenMinutes => self.start_of_minutes(us, 10), + Round::FifteenMinutes => self.start_of_minutes(us, 15), + Round::TimeSlot => self.start_of_minutes(us, 30), + Round::Hour => self.round_down(us, 3600), + Round::Day => self.round_down(us, 24 * 3600), + } } #[inline] - pub fn round_timestamp(&self, ts: i64, round: Round) -> i64 { - match round { - Round::Second => ts, - Round::Minute => self.start_of_minutes(ts, 60), - Round::FiveMinutes => self.start_of_minutes(ts, 5 * 60), - Round::TenMinutes => self.start_of_minutes(ts, 10 * 60), - Round::FifteenMinutes => self.start_of_minutes(ts, 15 * 60), - Round::TimeSlot => self.start_of_minutes(ts, 30 * 60), - Round::Hour => self.round_down(ts, 3600), - Round::Day => self.round_down(ts, 3600 * 24), + pub fn to_minute(&self, us: i64) -> u8 { + if us >= 0 && self.offset_round_hour { + ((us / MICROS_IN_A_SEC / 60) % 60) as u8 + } else { + let datetime = self.to_datetime_from_us(us); + datetime.minute() as u8 } } - pub fn to_minute(&self, ts: i64) -> u8 { - if ts >= 0 && self.offset_round_hour { - ((ts / 60) % 60) as u8 + #[inline] + pub fn to_second(&self, us: i64) -> u8 { + if us >= 0 { + (us / MICROS_IN_A_SEC % 60) as u8 } else { - let datetime = self.tz.timestamp(ts, 0); - datetime.minute() as u8 + let datetime = self.to_datetime_from_us(us); + datetime.second() as u8 } } - pub fn to_hour(&self, ts: i64) -> u8 { - let datetime = self.tz.timestamp(ts, 0); + #[inline] + pub fn to_datetime_from_us(&self, us: i64) -> DateTime { + us.to_timestamp(self.tz) + } + + #[inline] + pub fn to_hour(&self, us: i64) -> u8 { + let datetime = self.to_datetime_from_us(us); datetime.hour() as u8 } } @@ -313,11 +328,11 @@ macro_rules! impl_interval_year_month { } pub fn eval_timestamp( - ts: i64, + us: i64, tz: TzLUT, delta: impl AsPrimitive, ) -> Result { - let ts = ts.to_timestamp(tz.tz); + let ts = us.to_timestamp(tz.tz); let new_ts = $op(ts.year(), ts.month(), ts.day(), delta.as_())?; check_timestamp(NaiveDateTime::new(new_ts, ts.time()).timestamp_micros()) } @@ -351,11 +366,11 @@ impl AddTimesImpl { } pub fn eval_timestamp( - ts: i64, + us: i64, delta: impl AsPrimitive, factor: i64, ) -> Result { - check_timestamp(ts.wrapping_add(delta.as_() * factor * MICROS_IN_A_SEC)) + check_timestamp(us.wrapping_add(delta.as_() * factor * MICROS_IN_A_SEC)) } } @@ -374,8 +389,8 @@ pub trait ToNumber { pub struct ToNumberImpl; impl ToNumberImpl { - pub fn eval_timestamp, R>(ts: i64, tz: TzLUT) -> R { - let dt = ts.to_timestamp(tz.tz); + pub fn eval_timestamp, R>(us: i64, tz: TzLUT) -> R { + let dt = us.to_timestamp(tz.tz); T::to_number(&dt) } @@ -465,8 +480,8 @@ pub enum Round { pub struct DateRounder; impl DateRounder { - pub fn eval_timestamp>(ts: i64, tz: TzLUT) -> i32 { - let dt = ts.to_timestamp(tz.tz); + pub fn eval_timestamp>(us: i64, tz: TzLUT) -> i32 { + let dt = us.to_timestamp(tz.tz); T::to_number(&dt) } diff --git a/src/query/functions/src/scalars/datetime.rs b/src/query/functions/src/scalars/datetime.rs index 0a15a9d229756..153288f9c1195 100644 --- a/src/query/functions/src/scalars/datetime.rs +++ b/src/query/functions/src/scalars/datetime.rs @@ -901,23 +901,19 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_hour", FunctionProperty::default(), |_| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - ctx.tz.to_hour(val / MICROS_IN_A_SEC) - }), + vectorize_1_arg::(|val, ctx| ctx.tz.to_hour(val)), ); registry.register_passthrough_nullable_1_arg::( "to_minute", FunctionProperty::default(), |_| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - ctx.tz.to_minute(val / MICROS_IN_A_SEC) - }), + vectorize_1_arg::(|val, ctx| ctx.tz.to_minute(val)), ); registry.register_passthrough_nullable_1_arg::( "to_second", FunctionProperty::default(), |_| FunctionDomain::Full, - vectorize_1_arg::(|val, _ctx| (val / MICROS_IN_A_SEC % 60) as u8), + vectorize_1_arg::(|val, ctx| ctx.tz.to_second(val)), ); } @@ -1088,7 +1084,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::Second) + ctx.tz.round_us(val, Round::Second) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1096,7 +1092,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::Minute) + ctx.tz.round_us(val, Round::Minute) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1104,7 +1100,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::FiveMinutes) + ctx.tz.round_us(val, Round::FiveMinutes) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1112,7 +1108,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::TenMinutes) + ctx.tz.round_us(val, Round::TenMinutes) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1120,7 +1116,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::FifteenMinutes) + ctx.tz.round_us(val, Round::FifteenMinutes) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1128,7 +1124,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::Hour) + ctx.tz.round_us(val, Round::Hour) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1136,7 +1132,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::Day) + ctx.tz.round_us(val, Round::Day) }), ); registry.register_passthrough_nullable_1_arg::( @@ -1144,7 +1140,7 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { FunctionProperty::default(), |_| FunctionDomain::Full, vectorize_1_arg::(|val, ctx| { - ctx.tz.round_timestamp_micros(val, Round::TimeSlot) + ctx.tz.round_us(val, Round::TimeSlot) }), ); From c65b5b72e3fe99bc650b972e80b93a5819165cb4 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Feb 2023 22:24:21 +0800 Subject: [PATCH 7/7] update --- src/query/expression/src/utils/date_helper.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/expression/src/utils/date_helper.rs index f3303b1b14509..7de17a4583f3e 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/expression/src/utils/date_helper.rs @@ -195,7 +195,14 @@ impl TzLUT { Round::FifteenMinutes => self.start_of_minutes(us, 15), Round::TimeSlot => self.start_of_minutes(us, 30), Round::Hour => self.round_down(us, 3600), - Round::Day => self.round_down(us, 24 * 3600), + Round::Day => { + let dt = self.to_datetime_from_us(us); + let dt = self + .tz + .ymd(dt.year(), dt.month(), dt.day()) + .and_hms_micro(0, 0, 0, 0); + dt.timestamp() * MICROS_IN_A_SEC + } } }