planner, stats: skip (auto) analyze on vector column and index (#56584)

close #56583

pkg/executor/BUILD.bazel

@@ -414,7 +414,6 @@ go_test(
         "//pkg/planner/core/resolve",
         "//pkg/planner/property",
         "//pkg/planner/util",
-        "//pkg/planner/util/coretestsdk",
         "//pkg/server",
         "//pkg/session",
         "//pkg/session/types",

pkg/executor/partition_table_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/infoschema"
 	"github.com/pingcap/tidb/pkg/parser/model"
-	"github.com/pingcap/tidb/pkg/planner/util/coretestsdk"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/testkit/external"
 	"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
@@ -317,18 +316,18 @@ func TestOrderByAndLimit(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "trange")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "thash")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "tlist")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "tregular")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "trange_intpk")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "thash_intpk")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "tlist_intpk")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "tregular_intpk")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "trange_clustered")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "thash_clustered")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "tlist_clustered")
-	coretestsdk.SetTiFlashReplica(t, dom, "test_orderby_limit", "tregular_clustered")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "trange")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "thash")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "tlist")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "tregular")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "trange_intpk")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "thash_intpk")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "tlist_intpk")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "tregular_intpk")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "trange_clustered")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "thash_clustered")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "tlist_clustered")
+	testkit.SetTiFlashReplica(t, dom, "test_orderby_limit", "tregular_clustered")
 	tk.MustExec("set @@session.tidb_isolation_read_engines=\"tikv\"")
 
 	// test indexLookUp

pkg/planner/core/BUILD.bazel

@@ -290,7 +290,6 @@ go_test(
         "//pkg/planner/core/rule",
         "//pkg/planner/property",
         "//pkg/planner/util",
-        "//pkg/planner/util/coretestsdk",
         "//pkg/planner/util/costusage",
         "//pkg/planner/util/optimizetrace",
         "//pkg/session",

pkg/planner/core/casetest/BUILD.bazel

@@ -24,7 +24,6 @@ go_test(
         "//pkg/planner/core/operator/logicalop",
         "//pkg/planner/core/resolve",
         "//pkg/planner/property",
-        "//pkg/planner/util/coretestsdk",
         "//pkg/testkit",
         "//pkg/testkit/testdata",
         "//pkg/testkit/testmain",

pkg/planner/core/casetest/enforcempp/BUILD.bazel

@@ -14,7 +14,6 @@ go_test(
         "//pkg/domain",
         "//pkg/meta/model",
         "//pkg/parser/model",
-        "//pkg/planner/util/coretestsdk",
         "//pkg/store/mockstore",
         "//pkg/testkit",
         "//pkg/testkit/external",

pkg/planner/core/casetest/enforcempp/enforce_mpp_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/meta/model"
 	pmodel "github.com/pingcap/tidb/pkg/parser/model"
-	"github.com/pingcap/tidb/pkg/planner/util/coretestsdk"
 	"github.com/pingcap/tidb/pkg/store/mockstore"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/testkit/external"
@@ -56,9 +55,9 @@ func TestEnforceMPP(t *testing.T) {
 	is := dom.InfoSchema()
 	db, exists := is.SchemaByName(pmodel.NewCIStr("test"))
 	require.True(t, exists)
-	coretestsdk.SetTiFlashReplica(t, dom, db.Name.L, "t")
-	coretestsdk.SetTiFlashReplica(t, dom, db.Name.L, "s")
-	coretestsdk.SetTiFlashReplica(t, dom, db.Name.L, "t3")
+	testkit.SetTiFlashReplica(t, dom, db.Name.L, "t")
+	testkit.SetTiFlashReplica(t, dom, db.Name.L, "s")
+	testkit.SetTiFlashReplica(t, dom, db.Name.L, "t3")
 
 	var input []string
 	var output []struct {
@@ -143,7 +142,7 @@ func TestEnforceMPPWarning1(t *testing.T) {
 		if strings.HasPrefix(tt, "cmd: enable-replica") {
 			// Create virtual tiflash replica info.
 			dom := domain.GetDomain(tk.Session())
-			coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
+			testkit.SetTiFlashReplica(t, dom, "test", "t")
 			continue
 		}
 		testdata.OnRecord(func() {
@@ -277,8 +276,8 @@ func TestEnforceMPPWarning4(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "s")
+	testkit.SetTiFlashReplica(t, dom, "test", "t")
+	testkit.SetTiFlashReplica(t, dom, "test", "s")
 
 	var input []string
 	var output []struct {
@@ -329,9 +328,9 @@ func TestMPP2PhaseAggPushDown(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "c")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "o")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
+	testkit.SetTiFlashReplica(t, dom, "test", "c")
+	testkit.SetTiFlashReplica(t, dom, "test", "o")
+	testkit.SetTiFlashReplica(t, dom, "test", "t")
 
 	var input []string
 	var output []struct {

pkg/planner/core/casetest/hint/BUILD.bazel

@@ -15,7 +15,6 @@ go_test(
         "//pkg/domain",
         "//pkg/meta/model",
         "//pkg/parser/model",
-        "//pkg/planner/util/coretestsdk",
         "//pkg/sessionctx/variable",
         "//pkg/store/mockstore",
         "//pkg/testkit",

pkg/planner/core/casetest/hint/hint_test.go

@@ -21,7 +21,6 @@ import (
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/meta/model"
 	pmodel "github.com/pingcap/tidb/pkg/parser/model"
-	"github.com/pingcap/tidb/pkg/planner/util/coretestsdk"
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/store/mockstore"
 	"github.com/pingcap/tidb/pkg/testkit"
@@ -45,9 +44,9 @@ func TestReadFromStorageHint(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "tt")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "ttt")
+	testkit.SetTiFlashReplica(t, dom, "test", "t")
+	testkit.SetTiFlashReplica(t, dom, "test", "tt")
+	testkit.SetTiFlashReplica(t, dom, "test", "ttt")
 
 	var input []string
 	var output []struct {
@@ -154,9 +153,9 @@ func TestJoinHintCompatibility(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t4")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t5")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t6")
+	testkit.SetTiFlashReplica(t, dom, "test", "t4")
+	testkit.SetTiFlashReplica(t, dom, "test", "t5")
+	testkit.SetTiFlashReplica(t, dom, "test", "t6")
 
 	tk.MustExec("create definer='root'@'localhost' view v as select /*+ leading(t1), inl_join(t1) */ t.a from t join t1 join t2 where t.a = t1.a and t1.b = t2.b;")
 	tk.MustExec("create definer='root'@'localhost' view v1 as select /*+ leading(t2), merge_join(t) */ t.a from t join t1 join t2 where t.a = t1.a and t1.b = t2.b;")
@@ -194,7 +193,7 @@ func TestReadFromStorageHintAndIsolationRead(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
+	testkit.SetTiFlashReplica(t, dom, "test", "t")
 
 	var input []string
 	var output []struct {
@@ -227,7 +226,7 @@ func TestIsolationReadTiFlashUseIndexHint(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
+	testkit.SetTiFlashReplica(t, dom, "test", "t")
 
 	tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
 	var input []string

pkg/planner/core/casetest/index/BUILD.bazel

@@ -14,7 +14,6 @@ go_test(
         "//pkg/domain",
         "//pkg/domain/infosync",
         "//pkg/parser/model",
-        "//pkg/planner/util/coretestsdk",
         "//pkg/store/mockstore",
         "//pkg/testkit",
         "//pkg/testkit/testdata",

pkg/planner/core/casetest/index/index_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/domain/infosync"
 	"github.com/pingcap/tidb/pkg/parser/model"
-	"github.com/pingcap/tidb/pkg/planner/util/coretestsdk"
 	"github.com/pingcap/tidb/pkg/store/mockstore"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/testkit/testdata"
@@ -261,7 +260,7 @@ func TestVectorIndex(t *testing.T) {
 	tk.MustExec("alter table t set tiflash replica 1;")
 	tk.MustExec("alter table t add vector index vecIdx1((vec_cosine_distance(d))) USING HNSW;")
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
+	testkit.SetTiFlashReplica(t, dom, "test", "t")
 	tk.MustUseIndex("select * from t use index(vecIdx1) order by vec_cosine_distance(d, '[1,1,1,1]') limit 1", "vecIdx1")
 	tk.MustUseIndex("select * from t use index(vecIdx1) order by vec_cosine_distance('[1,1,1,1]', d) limit 1", "vecIdx1")
 	tk.MustExecToErr("select * from t use index(vecIdx1) order by vec_l2_distance(d, '[1,1,1,1]') limit 1")
@@ -282,12 +281,14 @@ func TestAnalyzeVectorIndex(t *testing.T) {
 		tiflash.Unlock()
 	}()
 	tk.MustExec(`create table t(a int, b vector(2), c vector(3), j json, index(a))`)
+	tk.MustExec("insert into t values(1, '[1, 0]', '[1, 0, 0]', '{\"a\": 1}')")
 	tk.MustExec("alter table t set tiflash replica 2 location labels 'a','b';")
-	tblInfo, err := dom.InfoSchema().TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr("t"))
+	tbl, err := dom.InfoSchema().TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr("t"))
 	require.NoError(t, err)
-	err = domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tblInfo.Meta().ID, true)
+	tblInfo := tbl.Meta()
+	err = domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tblInfo.ID, true)
 	require.NoError(t, err)
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t")
+	testkit.SetTiFlashReplica(t, dom, "test", "t")
 
 	testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/MockCheckVectorIndexProcess", `return(1)`)
 	tk.MustExec("alter table t add vector index idx((VEC_COSINE_DISTANCE(b))) USING HNSW")
@@ -306,12 +307,26 @@ func TestAnalyzeVectorIndex(t *testing.T) {
 		"Warning 1105 analyzing vector index is not supported, skip idx2"))
 	tk.MustExec("analyze table t index idx")
 	tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
-		"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t, reason to use this rate is \"TiDB assumes that the table is empty, use sample-rate=1\"",
+		"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t, reason to use this rate is \"use min(1, 110000/1) as the sample-rate=1\"",
 		"Warning 1105 No predicate column has been collected yet for table test.t, so only indexes and the columns composing the indexes will be analyzed",
 		"Warning 1105 The version 2 would collect all statistics not only the selected indexes",
 		"Warning 1105 analyzing vector index is not supported, skip idx",
 		"Warning 1105 analyzing vector index is not supported, skip idx2"))
 
+	statsHandle := dom.StatsHandle()
+	statsTbl := statsHandle.GetTableStats(tblInfo)
+	require.True(t, statsTbl.LastAnalyzeVersion > 0)
+	// int col
+	col := statsTbl.GetCol(1)
+	require.NotNil(t, col)
+	// It has stats.
+	require.True(t, (col.Histogram.Len()+col.TopN.Num()) > 0)
+	// vec col
+	col = statsTbl.GetCol(2)
+	require.NotNil(t, col)
+	// It doesn't have stats.
+	require.False(t, (col.Histogram.Len()+col.TopN.Num()) > 0)
+
 	tk.MustExec("set tidb_analyze_version=1")
 	tk.MustExec("analyze table t")
 	tk.MustQuery("show warnings").Sort().Check(testkit.Rows(

pkg/planner/core/casetest/integration_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/meta/model"
 	pmodel "github.com/pingcap/tidb/pkg/parser/model"
-	"github.com/pingcap/tidb/pkg/planner/util/coretestsdk"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/testkit/testdata"
 	"github.com/stretchr/testify/require"
@@ -57,8 +56,8 @@ func TestVerboseExplain(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t1")
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "t2")
+	testkit.SetTiFlashReplica(t, dom, "test", "t1")
+	testkit.SetTiFlashReplica(t, dom, "test", "t2")
 
 	var input []string
 	var output []struct {
@@ -144,7 +143,7 @@ func TestMergeContinuousSelections(t *testing.T) {
 
 	// Create virtual tiflash replica info.
 	dom := domain.GetDomain(tk.Session())
-	coretestsdk.SetTiFlashReplica(t, dom, "test", "ts")
+	testkit.SetTiFlashReplica(t, dom, "test", "ts")
 
 	tk.MustExec(" set @@tidb_allow_mpp=1;")
 

pkg/planner/core/casetest/mpp/BUILD.bazel

@@ -15,7 +15,6 @@ go_test(
         "//pkg/domain",
         "//pkg/meta/model",
         "//pkg/parser/model",
-        "//pkg/planner/util/coretestsdk",
         "//pkg/testkit",
         "//pkg/testkit/testdata",
         "//pkg/testkit/testmain",