Skip to content

[Fix](timezone) Fix incorrect DST handling #51454

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhangstar333 merged 3 commits into from
Jun 24, 2025
Merged

[Fix](timezone) Fix incorrect DST handling #51454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c43ad6f
1
zclllyybb Jun 3, 2025
783ea73
fix
zclllyybb Jun 23, 2025
de0ee4d
fix_compile_gcc
zclllyybb Jun 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion be/src/olap/rowset/segment_v2/inverted_index/util/priority_queue.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ template <typename T>
class PriorityQueue {
public:
PriorityQueue(size_t max_size, std::function<T()> sentinel_object_supplier = nullptr) {
assert(max_size >= 0 && max_size < std::numeric_limits<int32_t>::max());
assert(max_size < std::numeric_limits<int32_t>::max());
size_t heap_size = (max_size == 0) ? 2 : max_size + 1;
_heap.resize(heap_size);
_max_size = max_size;
Expand Down
57 changes: 23 additions & 34 deletions be/src/vec/runtime/vdatetime_value.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2020,7 +2020,7 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale

int field_idx = 0;
int field_len = year_len;
long sec_offset = 0;
int sec_offset = 0;
bool need_use_timezone = false;

while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) {
Expand Down Expand Up @@ -2185,40 +2185,29 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale
if (!TimezoneUtils::find_cctz_time_zone(std::string {ptr, end}, given_tz)) {
return false; // invalid format
}
auto given = cctz::convert(cctz::civil_second {}, given_tz);
auto local = cctz::convert(cctz::civil_second {}, *local_time_zone);
// these two values is absolute time. so they are negative. need to use (-local) - (-given)
sec_offset = std::chrono::duration_cast<std::chrono::seconds>(given - local).count();
}

// In check_range_and_set_time, for Date type the time part will be truncated. So if the timezone offset should make
// rounding to date part, it would be lost. To avoid this, we use a Datetime type to do these calc. It will save the
// time part and apply the offset. Then convert to Date type back.
// see https://github.com/apache/doris/pull/33553 for more details.
if constexpr (!is_datetime) {
if (sec_offset) {
DateV2Value<DateTimeV2ValueType> tmp;
if (!tmp.check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3],
date_val[4], date_val[5], date_val[6])) {
return false;
}
if (!tmp.date_add_interval<TimeUnit::SECOND>(
TimeInterval {TimeUnit::SECOND, sec_offset, false})) {
return false;
}
this->assign_from(tmp);
return true;
if (is_invalid(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4], date_val[5],
date_val[6])) {
return false;
}
}

if (!check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4],
date_val[5], date_val[6])) {
return false;
}

return sec_offset ? date_add_interval<TimeUnit::SECOND>(
TimeInterval {TimeUnit::SECOND, sec_offset, false})
: true;
// will carring on the bits in cctz::civil_second. if day is 70, will carry to month.
cctz::civil_second cs {date_val[0], date_val[1], date_val[2],
date_val[3], date_val[4], date_val[5]};

auto given = cctz::convert(cs, given_tz);
auto local = cctz::convert(given, *local_time_zone);
date_val[0] = local.year();
date_val[1] = local.month();
date_val[2] = local.day();
date_val[3] = local.hour();
date_val[4] = local.minute();
date_val[5] = local.second();
}

return check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4],
date_val[5], date_val[6]) &&
(sec_offset ? date_add_interval<TimeUnit::SECOND>(
TimeInterval {TimeUnit::SECOND, sec_offset, false})
: true);
}

template <typename T>
Expand Down
20 changes: 10 additions & 10 deletions be/src/vec/runtime/vdatetime_value.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1065,8 +1065,8 @@ class DateV2Value {
}

bool operator==(const VecDateTimeValue& other) const {
int64_t ts1;
int64_t ts2;
int64_t ts1 = 0;
int64_t ts2 = 0;
this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
return ts1 == ts2;
Expand All @@ -1081,8 +1081,8 @@ class DateV2Value {
bool operator<=(const DateV2Value<T>& other) const { return !(*this > other); }

bool operator<=(const VecDateTimeValue& other) const {
int64_t ts1;
int64_t ts2;
int64_t ts1 = 0;
int64_t ts2 = 0;
this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
return ts1 <= ts2;
Expand All @@ -1091,8 +1091,8 @@ class DateV2Value {
bool operator>=(const DateV2Value<T>& other) const { return !(*this < other); }

bool operator>=(const VecDateTimeValue& other) const {
int64_t ts1;
int64_t ts2;
int64_t ts1 = 0;
int64_t ts2 = 0;
this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
return ts1 >= ts2;
Expand All @@ -1103,8 +1103,8 @@ class DateV2Value {
}

bool operator<(const VecDateTimeValue& other) const {
int64_t ts1;
int64_t ts2;
int64_t ts1 = 0;
int64_t ts2 = 0;
this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
return ts1 < ts2;
Expand All @@ -1115,8 +1115,8 @@ class DateV2Value {
}

bool operator>(const VecDateTimeValue& other) const {
int64_t ts1;
int64_t ts2;
int64_t ts1 = 0;
int64_t ts2 = 0;
this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
return ts1 > ts2;
Expand Down
8 changes: 5 additions & 3 deletions be/test/vec/function/cast/cast_to_date_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@ TEST_F(FunctionCastTest, strict_test_from_string_to_date) {
// Valid ISO 8601 format with timezone
{{std::string("2023-07-16T19:20:30.123+08:00")}, std::string("2023-07-16")},
{{std::string("2023-07-16T19+08:00")}, std::string("2023-07-16")},
{{std::string("2023-07-16T1920+08:00")}, std::string("2023-07-16")},
// keep its origin behaviour. timezone offset and time part could together impact date part.
// here 1920 is treated as 1,920 hours and carried over to the day.
{{std::string("2023-07-16T1920+08:00")}, std::string("2023-10-04")},
{{std::string("70-1-1T00:00:00-0000")}, std::string("1970-01-01")},
{{std::string("19991231T235960.5UTC")}, Null()},
{{std::string("19991231T235960.5UTC")}, std::string("2000-01-01")},

// Date with timezone names
{{std::string("2024-02-29 12:00:00 Europe/Paris")}, std::string("2024-02-29")},
Expand Down Expand Up @@ -101,7 +103,7 @@ TEST_F(FunctionCastTest, non_strict_test_from_string_to_date) {
{{std::string("0023-1-1T1:2:3. -00:00")}, std::string("0023-01-01")},
{{std::string("2025/06/15T00:00:00.0-0")}, std::string("2025-06-15")},
{{std::string("2025/06/15T00:00:00.99999999999")}, std::string("2025-06-15")},
{{std::string("2024-02-29T23-59-60ZULU")}, Null()},
{{std::string("2024-02-29T23-59-60ZULU")}, std::string("2024-03-01")},
{{std::string("2024 12 31T121212.123456 America/New_York")}, Null()},

// Invalid formats
Expand Down
3 changes: 2 additions & 1 deletion be/test/vec/function/cast/cast_to_datetime_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ TEST_F(FunctionCastTest, strict_test_from_string_to_datetime) {

// Various timezone offsets
{{std::string("2020-12-12 13:12:12-03:00")}, std::string("2020-12-13 00:12:12")},
{{std::string("0023-01-01T00:00Z")}, std::string("0023-01-01 08:00:00")},
// CST of shanghai before 1900 is not +080000 but +080543
{{std::string("0023-01-01T00:00Z")}, std::string("0023-01-01 08:05:43")},

// Year cutoff cases
{{std::string("69-12-31")}, std::string("2069-12-31")},
Expand Down
11 changes: 3 additions & 8 deletions be/test/vec/function/function_test_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,15 +418,10 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty
<< ", expected result: " << result_type_ptr->to_string(*expected_col_ptr, i);
} else {
auto comp_res = column->compare_at(i, i, *expected_col_ptr, 1);
if (0 != comp_res) {
std::cerr << "function " << func_name << " result mismatch, row " << i << ":\n"
<< block.dump_data(i, 1) << std::endl
<< ", expected result: "
<< result_type_ptr->to_string(*expected_col_ptr, i) << std::endl;
}
EXPECT_EQ(0, comp_res)
<< ", function " << func_name
<< " result: " << block.get_data_types()[result]->to_string(*column, i)
<< ", function " << func_name << ". input row:\n"
<< block.dump_data(i, 1)
<< "result: " << block.get_data_types()[result]->to_string(*column, i)
<< ", expected result: " << result_type_ptr->to_string(*expected_col_ptr, i);
}
}
Expand Down
8 changes: 4 additions & 4 deletions regression-test/data/cast_p0/cast_to_datetime.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
2020-12-13T00:12:12

-- !sql --
0023-01-01T08:00
0023-01-01T08:05:43

-- !sql --
2069-12-31T00:00
Expand Down Expand Up @@ -243,7 +243,7 @@
2020-12-13T00:12:12

-- !sql --
0023-01-01T08:00
0023-01-01T08:05:43

-- !sql --
2069-12-31T00:00
Expand Down Expand Up @@ -372,7 +372,7 @@
1970-01-01T00:00

-- !sql --
2000-01-01T07:59:59
2000-01-01T08:00

-- !sql --
2024-05-01T00:00
Expand Down Expand Up @@ -423,7 +423,7 @@
2020-12-13T00:12:12

-- !sql --
0023-01-01T08:00
0023-01-01T08:05:43

-- !sql --
2069-12-31T00:00
Expand Down
16 changes: 6 additions & 10 deletions regression-test/data/datatype_p0/datetimev2/test_timezone.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !legacy --
2022-01-01T01:02:55 2022-01-01
2022-02-01T03:02:55 2022-02-01
2022-02-28T19:02:55 2022-03-01
2022-04-01T09:02:55 2022-03-31
2022-05-01T00:32:55 2022-05-01
2022-05-31T22:32:55 2022-06-01
2022-06-30T20:02:55 2022-07-01
2022-07-31T21:00 2022-08-01

-- !nereids --
2022-01-01T01:02:55 2022-01-01
2022-02-01T03:02:55 2022-02-01
Expand All @@ -25,3 +15,9 @@
-- !fold3 --
2020-12-12T13:12:12

-- !nodst --
2010-01-05T10:15:30

-- !dst --
2010-08-05T09:15:30

2 changes: 1 addition & 1 deletion regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
3 2023-08-17T17:41:18
4 2023-08-17T14:41:18
5 2023-08-17T09:41:18
6 2023-08-18T01:41:18
6 2023-08-18T00:41:18
7 2023-08-17T17:41:18
8 2023-08-17T19:41:18

Expand Down
5 changes: 5 additions & 0 deletions regression-test/suites/datatype_p0/datetimev2/test_timezone.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,9 @@ suite("test_timezone") {
qt_fold1 """ select cast('2020-12-12T12:12:12asia/shanghai' as datetime); """
qt_fold2 """ select cast('2020-12-12T12:12:12america/los_angeLES' as datetime); """
qt_fold3 """ select cast('2020-12-12T12:12:12Europe/pARIS' as datetime); """

qt_nodst "select cast('2010-01-05 08:15:30Europe/London' as datetime);"
qt_dst "select cast('2010-08-05 08:15:30Europe/London' as datetime);"
testFoldConst ("select cast('2010-01-05 08:15:30Europe/London' as datetime);")
testFoldConst ("select cast('2010-08-05 08:15:30Europe/London' as datetime);")
}
Loading