planner: adjust N used in TopN cost formula based on the total number…

… of rows (#46368) (#46570)

close #43285

planner/core/plan_cost_ver2.go

@@ -388,6 +388,11 @@ func (p *PhysicalTopN) getPlanCostVer2(taskType property.TaskType, option *PlanC
 	rows := getCardinality(p.children[0], option.CostFlag)
 	N := math.Max(1, float64(p.Count+p.Offset))
 	rowSize := getAvgRowSize(p.statsInfo(), p.Schema().Columns)
+	if N > 10000 {
+		// It's only used to prevent some extreme cases, e.g. `select * from t order by a limit 18446744073709551615`.
+		// For normal cases, considering that `rows` may be under-estimated, better to keep `n` unchanged.
+		N = math.Min(N, rows)
+	}
 	cpuFactor := getTaskCPUFactorVer2(p, taskType)
 	memFactor := getTaskMemFactorVer2(p, taskType)
 

planner/core/plan_cost_ver2_test.go

@@ -53,6 +53,18 @@ func testCostQueries(t *testing.T, tk *testkit.TestKit, queries []string) {
 	}
 }
 
+func TestHugeTopNCost(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec(`create table t (a int)`)
+	tk.MustExec(`insert into t values (1)`)
+	tk.MustExec(`analyze table t`)
+	plan1 := tk.MustQuery("explain format='verbose' select /*+ limit_to_cop() */ * from t where a=1 order by a limit 1")
+	plan2 := tk.MustQuery("explain format='verbose' select /*+ limit_to_cop() */ * from t where a=1 order by a limit 1000000000")
+	require.Equal(t, plan1.Rows()[0][2], plan2.Rows()[0][2]) // should have the same plan cost
+}
+
 func TestCostModelVer2(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)