From b2fe09bc3f5de943679f3de963aaa910bd0dd415 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 23 Aug 2023 20:05:52 +0800 Subject: [PATCH 1/6] fixup --- planner/core/plan_cost_ver2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/planner/core/plan_cost_ver2.go b/planner/core/plan_cost_ver2.go index c9fd0497a1289..3575c98ec9e02 100644 --- a/planner/core/plan_cost_ver2.go +++ b/planner/core/plan_cost_ver2.go @@ -396,6 +396,7 @@ func (p *PhysicalTopN) getPlanCostVer2(taskType property.TaskType, option *PlanC rows := getCardinality(p.children[0], option.CostFlag) n := math.Max(1, float64(p.Count+p.Offset)) + n = math.Min(n, rows) rowSize := getAvgRowSize(p.StatsInfo(), p.Schema().Columns) cpuFactor := getTaskCPUFactorVer2(p, taskType) memFactor := getTaskMemFactorVer2(p, taskType) From cd1a896c95d0ce89e4cfcfef1befedd7cd662bd3 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 31 Aug 2023 15:49:21 +0800 Subject: [PATCH 2/6] make linter happy --- planner/core/integration_test.go | 37 ++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index b5fc2c7a06591..1cfcdc7646de7 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -3205,31 +3205,40 @@ func TestLimitPushDown(t *testing.T) { tk.MustExec(`create table t (a int)`) tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1)`) tk.MustExec(`analyze table t`) tk.MustExec(`set tidb_opt_limit_push_down_threshold=0`) tk.MustQuery(`explain format=brief select a from t order by a desc limit 10`).Check(testkit.Rows( - `TopN 1.00 root test.t.a:desc, offset:0, count:10`, - `└─TableReader 1.00 root data:TableFullScan`, - ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) + `TopN 10.00 root test.t.a:desc, offset:0, count:10`, + `└─TableReader 10.00 root data:TableFullScan`, + ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) tk.MustExec(`set tidb_opt_limit_push_down_threshold=10`) tk.MustQuery(`explain format=brief select a from t order by a desc limit 10`).Check(testkit.Rows( - `TopN 1.00 root test.t.a:desc, offset:0, count:10`, - `└─TableReader 1.00 root data:TopN`, - ` └─TopN 1.00 cop[tikv] test.t.a:desc, offset:0, count:10`, - ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) + `TopN 10.00 root test.t.a:desc, offset:0, count:10`, + `└─TableReader 10.00 root data:TopN`, + ` └─TopN 10.00 cop[tikv] test.t.a:desc, offset:0, count:10`, + ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) tk.MustQuery(`explain format=brief select a from t order by a desc limit 11`).Check(testkit.Rows( - `TopN 1.00 root test.t.a:desc, offset:0, count:11`, - `└─TableReader 1.00 root data:TableFullScan`, - ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) + `TopN 10.00 root test.t.a:desc, offset:0, count:11`, + `└─TableReader 10.00 root data:TableFullScan`, + ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) tk.MustQuery(`explain format=brief select /*+ limit_to_cop() */ a from t order by a desc limit 11`).Check(testkit.Rows( - `TopN 1.00 root test.t.a:desc, offset:0, count:11`, - `└─TableReader 1.00 root data:TopN`, - ` └─TopN 1.00 cop[tikv] test.t.a:desc, offset:0, count:11`, - ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) + `TopN 10.00 root test.t.a:desc, offset:0, count:11`, + `└─TableReader 10.00 root data:TopN`, + ` └─TopN 10.00 cop[tikv] test.t.a:desc, offset:0, count:11`, + ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) } func TestIssue26559(t *testing.T) { From 68958f90f6d8a97b0bc0612cb4c0d9ea8d66ff13 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 31 Aug 2023 15:50:40 +0800 Subject: [PATCH 3/6] make linter happy --- planner/core/plan_cost_ver2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/plan_cost_ver2.go b/planner/core/plan_cost_ver2.go index 3575c98ec9e02..2df6035d44843 100644 --- a/planner/core/plan_cost_ver2.go +++ b/planner/core/plan_cost_ver2.go @@ -395,8 +395,8 @@ func (p *PhysicalTopN) getPlanCostVer2(taskType property.TaskType, option *PlanC } rows := getCardinality(p.children[0], option.CostFlag) - n := math.Max(1, float64(p.Count+p.Offset)) - n = math.Min(n, rows) + n := max(1, float64(p.Count+p.Offset)) + n = min(n, rows) rowSize := getAvgRowSize(p.StatsInfo(), p.Schema().Columns) cpuFactor := getTaskCPUFactorVer2(p, taskType) memFactor := getTaskMemFactorVer2(p, taskType) From 37c21c93c77b1f055572f16391874ed078517e94 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 31 Aug 2023 17:05:28 +0800 Subject: [PATCH 4/6] make linter happy --- planner/core/integration_test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 1cfcdc7646de7..ecb6c16378fc9 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -3204,16 +3204,7 @@ func TestLimitPushDown(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec(`create table t (a int)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) - tk.MustExec(`insert into t values (1)`) + tk.MustExec(`insert into t values (1), (1), (1), (1), (1), (1), (1), (1), (1), (1)`) tk.MustExec(`analyze table t`) tk.MustExec(`set tidb_opt_limit_push_down_threshold=0`) From 4911791b259b196eb5ea3b6aa9958a15437ef753 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 31 Aug 2023 17:55:52 +0800 Subject: [PATCH 5/6] make linter happy --- planner/core/integration_test.go | 30 +++++++++++++++--------------- planner/core/plan_cost_ver2.go | 6 +++++- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index ecb6c16378fc9..b5fc2c7a06591 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -3204,32 +3204,32 @@ func TestLimitPushDown(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec(`create table t (a int)`) - tk.MustExec(`insert into t values (1), (1), (1), (1), (1), (1), (1), (1), (1), (1)`) + tk.MustExec(`insert into t values (1)`) tk.MustExec(`analyze table t`) tk.MustExec(`set tidb_opt_limit_push_down_threshold=0`) tk.MustQuery(`explain format=brief select a from t order by a desc limit 10`).Check(testkit.Rows( - `TopN 10.00 root test.t.a:desc, offset:0, count:10`, - `└─TableReader 10.00 root data:TableFullScan`, - ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) + `TopN 1.00 root test.t.a:desc, offset:0, count:10`, + `└─TableReader 1.00 root data:TableFullScan`, + ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) tk.MustExec(`set tidb_opt_limit_push_down_threshold=10`) tk.MustQuery(`explain format=brief select a from t order by a desc limit 10`).Check(testkit.Rows( - `TopN 10.00 root test.t.a:desc, offset:0, count:10`, - `└─TableReader 10.00 root data:TopN`, - ` └─TopN 10.00 cop[tikv] test.t.a:desc, offset:0, count:10`, - ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) + `TopN 1.00 root test.t.a:desc, offset:0, count:10`, + `└─TableReader 1.00 root data:TopN`, + ` └─TopN 1.00 cop[tikv] test.t.a:desc, offset:0, count:10`, + ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) tk.MustQuery(`explain format=brief select a from t order by a desc limit 11`).Check(testkit.Rows( - `TopN 10.00 root test.t.a:desc, offset:0, count:11`, - `└─TableReader 10.00 root data:TableFullScan`, - ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) + `TopN 1.00 root test.t.a:desc, offset:0, count:11`, + `└─TableReader 1.00 root data:TableFullScan`, + ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) tk.MustQuery(`explain format=brief select /*+ limit_to_cop() */ a from t order by a desc limit 11`).Check(testkit.Rows( - `TopN 10.00 root test.t.a:desc, offset:0, count:11`, - `└─TableReader 10.00 root data:TopN`, - ` └─TopN 10.00 cop[tikv] test.t.a:desc, offset:0, count:11`, - ` └─TableFullScan 10.00 cop[tikv] table:t keep order:false`)) + `TopN 1.00 root test.t.a:desc, offset:0, count:11`, + `└─TableReader 1.00 root data:TopN`, + ` └─TopN 1.00 cop[tikv] test.t.a:desc, offset:0, count:11`, + ` └─TableFullScan 1.00 cop[tikv] table:t keep order:false`)) } func TestIssue26559(t *testing.T) { diff --git a/planner/core/plan_cost_ver2.go b/planner/core/plan_cost_ver2.go index 2df6035d44843..11fe20db5d2e8 100644 --- a/planner/core/plan_cost_ver2.go +++ b/planner/core/plan_cost_ver2.go @@ -396,7 +396,11 @@ func (p *PhysicalTopN) getPlanCostVer2(taskType property.TaskType, option *PlanC rows := getCardinality(p.children[0], option.CostFlag) n := max(1, float64(p.Count+p.Offset)) - n = min(n, rows) + if n > 10000 { + // It's only used to prevent some extreme cases, e.g. `select * from t order by a limit 18446744073709551615`. + // For normal cases, considering that `rows` may be under-estimated, better to keep `n` unchanged. + n = min(n, rows) + } rowSize := getAvgRowSize(p.StatsInfo(), p.Schema().Columns) cpuFactor := getTaskCPUFactorVer2(p, taskType) memFactor := getTaskMemFactorVer2(p, taskType) From 0bf2e679811a50c6f8e12d74dcbd464d48d23f81 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 31 Aug 2023 18:02:44 +0800 Subject: [PATCH 6/6] make linter happy --- planner/core/plan_cost_ver2_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/planner/core/plan_cost_ver2_test.go b/planner/core/plan_cost_ver2_test.go index 20d6e9f86a71b..275ccbeb80f1d 100644 --- a/planner/core/plan_cost_ver2_test.go +++ b/planner/core/plan_cost_ver2_test.go @@ -53,6 +53,18 @@ func testCostQueries(t *testing.T, tk *testkit.TestKit, queries []string) { } } +func TestHugeTopNCost(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(`create table t (a int)`) + tk.MustExec(`insert into t values (1)`) + tk.MustExec(`analyze table t`) + plan1 := tk.MustQuery("explain format='verbose' select /*+ limit_to_cop() */ * from t where a=1 order by a limit 1") + plan2 := tk.MustQuery("explain format='verbose' select /*+ limit_to_cop() */ * from t where a=1 order by a limit 1000000000") + require.Equal(t, plan1.Rows()[0][2], plan2.Rows()[0][2]) // should have the same plan cost +} + func TestCostModelVer2(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store)