From 2e2b84309c393bd5b0225e39e8728ad8fec4df59 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Wed, 21 Dec 2022 15:08:53 +0800 Subject: [PATCH 1/3] fix incorrect datetime value when loading stats --- statistics/handle/handle.go | 2 +- statistics/handle/handle_test.go | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 2b07e9aebcb99..e1b4dfe03fd89 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -1316,7 +1316,7 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID lowerBound = rows[i].GetDatum(2, &fields[2].Column.FieldType) upperBound = rows[i].GetDatum(3, &fields[3].Column.FieldType) } else { - sc := &stmtctx.StatementContext{TimeZone: time.UTC} + sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} d := rows[i].GetDatum(2, &fields[2].Column.FieldType) // When there's new collation data, the length of bounds of histogram(the collate key) might be // longer than the FieldType.Flen of this column. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 90fca36da41fc..45668e64f9ce5 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3137,3 +3137,42 @@ func (s *testStatsSuite) TestIncrementalModifyCountUpdate(c *C) { c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/injectBaseCount"), IsNil) c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/injectBaseModifyCount"), IsNil) } + +func TestIssue39336(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(` +create table t1 ( + a datetime(3) default null, + b int +) partition by range (b) ( + partition p0 values less than (1000), + partition p1 values less than (2000), + partition p3 values less than (maxvalue) +)`) + tk.MustExec("set @@sql_mode=''") + tk.MustExec(` +insert into t1 values +('2022-11-23 14:25:08.000', 1001), +('1000-00-09 00:00:00.000', 1001), +('1000-00-06 00:00:00.000', 1001), +('1000-00-06 00:00:00.000', 1001), +('2022-11-23 14:24:30.000', 1), +('2022-11-23 14:24:32.000', 1), +('2022-11-23 14:24:33.000', 1), +('2022-11-23 14:24:35.000', 1), +('1000-00-09 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('2022-11-23 14:25:11.000', 2001), +('2022-11-23 14:25:16.000', 3001), +('1000-00-09 00:00:00.000', 3001), +('1000-00-09 00:00:00.000', 2001), +('1000-00-06 00:00:00.000', 2001), +('1000-00-09 00:00:00.000', 2001)`) + tk.MustExec("analyze table t1") + rows := tk.MustQuery("show analyze status where job_info like 'merge global stats%'").Rows() + require.Len(t, rows, 1) + require.Equal(t, "finished", rows[0][7]) +} From 720b4cf2b763465ab3df51deb9fc2eb088867818 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 23 Dec 2022 14:02:36 +0800 Subject: [PATCH 2/3] upd --- statistics/handle/handle.go | 6 +++++- statistics/handle/handle_test.go | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index e1b4dfe03fd89..1e147218f1d4a 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -1316,7 +1316,11 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID lowerBound = rows[i].GetDatum(2, &fields[2].Column.FieldType) upperBound = rows[i].GetDatum(3, &fields[3].Column.FieldType) } else { - sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} + // Invalid date values or zero date values may be inserted into table under some relaxed sql mode. Those values + // may exist in statistics. Hence, when reading statistics, we should skip invalid date check and zero date check. + // See #39336. + //sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} + sc := &stmtctx.StatementContext{TimeZone: time.UTC} d := rows[i].GetDatum(2, &fields[2].Column.FieldType) // When there's new collation data, the length of bounds of histogram(the collate key) might be // longer than the FieldType.Flen of this column. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 45668e64f9ce5..29719b294a806 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3152,6 +3152,8 @@ create table t1 ( partition p3 values less than (maxvalue) )`) tk.MustExec("set @@sql_mode=''") + tk.MustExec("set @@tidb_analyze_version=2") + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") tk.MustExec(` insert into t1 values ('2022-11-23 14:25:08.000', 1001), @@ -3171,7 +3173,7 @@ insert into t1 values ('1000-00-09 00:00:00.000', 2001), ('1000-00-06 00:00:00.000', 2001), ('1000-00-09 00:00:00.000', 2001)`) - tk.MustExec("analyze table t1") + tk.MustExec("analyze table t1 with 0 topn") rows := tk.MustQuery("show analyze status where job_info like 'merge global stats%'").Rows() require.Len(t, rows, 1) require.Equal(t, "finished", rows[0][7]) From 68d8282543cb123e07be71ff6c1f4019bf7f144e Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 23 Dec 2022 16:07:31 +0800 Subject: [PATCH 3/3] upd --- statistics/handle/handle.go | 8 +++----- statistics/handle/handle_test.go | 21 ++++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 1e147218f1d4a..5cbad8fcacd87 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -1316,11 +1316,9 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID lowerBound = rows[i].GetDatum(2, &fields[2].Column.FieldType) upperBound = rows[i].GetDatum(3, &fields[3].Column.FieldType) } else { - // Invalid date values or zero date values may be inserted into table under some relaxed sql mode. Those values - // may exist in statistics. Hence, when reading statistics, we should skip invalid date check and zero date check. - // See #39336. - //sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} - sc := &stmtctx.StatementContext{TimeZone: time.UTC} + // Invalid date values may be inserted into table under some relaxed sql mode. Those values may exist in statistics. + // Hence, when reading statistics, we should skip invalid date check. See #39336. + sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} d := rows[i].GetDatum(2, &fields[2].Column.FieldType) // When there's new collation data, the length of bounds of histogram(the collate key) might be // longer than the FieldType.Flen of this column. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 29719b294a806..faf108d5b1ef0 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3148,31 +3148,22 @@ create table t1 ( b int ) partition by range (b) ( partition p0 values less than (1000), - partition p1 values less than (2000), - partition p3 values less than (maxvalue) + partition p1 values less than (maxvalue) )`) tk.MustExec("set @@sql_mode=''") tk.MustExec("set @@tidb_analyze_version=2") tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") tk.MustExec(` insert into t1 values -('2022-11-23 14:25:08.000', 1001), -('1000-00-09 00:00:00.000', 1001), -('1000-00-06 00:00:00.000', 1001), -('1000-00-06 00:00:00.000', 1001), +('1000-00-09 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), ('2022-11-23 14:24:30.000', 1), ('2022-11-23 14:24:32.000', 1), ('2022-11-23 14:24:33.000', 1), ('2022-11-23 14:24:35.000', 1), -('1000-00-09 00:00:00.000', 1), -('1000-00-06 00:00:00.000', 1), -('1000-00-06 00:00:00.000', 1), -('2022-11-23 14:25:11.000', 2001), -('2022-11-23 14:25:16.000', 3001), -('1000-00-09 00:00:00.000', 3001), -('1000-00-09 00:00:00.000', 2001), -('1000-00-06 00:00:00.000', 2001), -('1000-00-09 00:00:00.000', 2001)`) +('2022-11-23 14:25:08.000', 1001), +('2022-11-23 14:25:09.000', 1001)`) tk.MustExec("analyze table t1 with 0 topn") rows := tk.MustQuery("show analyze status where job_info like 'merge global stats%'").Rows() require.Len(t, rows, 1)