planner: move more methods from StatsHandle to its sub-packages (#47749…

…) (#47872)

ref #46905

pkg/executor/BUILD.bazel

@@ -165,6 +165,7 @@ go_library(
  "//pkg/statistics",
  "//pkg/statistics/handle",
  "//pkg/statistics/handle/cache",
+ "//pkg/statistics/handle/globalstats",
  "//pkg/statistics/handle/util",
  "//pkg/store/driver/backoff",
  "//pkg/store/driver/txn",

pkg/executor/analyze_col.go

@@ -280,8 +280,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats boo
  fms = append(fms, collectors[i].FMSketch)
  }
  if needExtStats {
- statsHandle := domain.GetDomain(e.ctx).StatsHandle()
- extStats, err = statsHandle.BuildExtendedStats(e.TableID.GetStatisticsID(), e.colsInfo, collectors)
+ extStats, err = statistics.BuildExtendedStats(e.ctx, e.TableID.GetStatisticsID(), e.colsInfo, collectors)
  if err != nil {
  return nil, nil, nil, nil, nil, err
  }

pkg/executor/analyze_col_v2.go

@@ -465,8 +465,7 @@ func (e *AnalyzeColumnsExecV2) buildSamplingStats(
 
  count = rootRowCollector.Base().Count
  if needExtStats {
- statsHandle := domain.GetDomain(e.ctx).StatsHandle()
- extStats, err = statsHandle.BuildExtendedStats(e.TableID.GetStatisticsID(), e.colsInfo, sampleCollectors)
+ extStats, err = statistics.BuildExtendedStats(e.ctx, e.TableID.GetStatisticsID(), e.colsInfo, sampleCollectors)
  if err != nil {
  return 0, nil, nil, nil, nil, err
  }

pkg/executor/analyze_global_stats.go

@@ -21,6 +21,7 @@ import (
  "github.com/pingcap/tidb/pkg/domain"
  "github.com/pingcap/tidb/pkg/infoschema"
  "github.com/pingcap/tidb/pkg/statistics"
+ "github.com/pingcap/tidb/pkg/statistics/handle/globalstats"
  "github.com/pingcap/tidb/pkg/statistics/handle/util"
  "github.com/pingcap/tidb/pkg/types"
  "github.com/pingcap/tidb/pkg/util/logutil"
@@ -84,7 +85,7 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob
  cache = nil
  }
 
- globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(
+ globalStatsI, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(
  e.Ctx(),
  globalOpts, e.Ctx().GetInfoSchema().(infoschema.InfoSchema),
  globalStatsID.tableID,
@@ -101,6 +102,7 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob
  }
  return err
  }
+ globalStats := globalStatsI.(*globalstats.GlobalStats)
  // Dump global-level stats to kv.
  for i := 0; i < globalStats.Num; i++ {
  hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i]

pkg/statistics/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
  "analyze.go",
  "analyze_jobs.go",
  "builder.go",
+ "builder_ext_stats.go",
  "cmsketch.go",
  "cmsketch_util.go",
  "column.go",

pkg/statistics/handle/extstats/extended_stats.go → pkg/statistics/builder_ext_stats.go

@@ -12,38 +12,42 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package extstats
+package statistics
 
 import (
+ "context"
  "encoding/json"
 
  "github.com/pingcap/errors"
+ "github.com/pingcap/tidb/pkg/kv"
  "github.com/pingcap/tidb/pkg/parser/ast"
  "github.com/pingcap/tidb/pkg/parser/model"
  "github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/statistics"
- "github.com/pingcap/tidb/pkg/statistics/handle/util"
  "github.com/pingcap/tidb/pkg/util/logutil"
- "github.com/pingcap/tidb/pkg/util/mathutil"
+ "github.com/pingcap/tidb/pkg/util/sqlexec"
  "go.uber.org/zap"
 )
 
 // BuildExtendedStats build extended stats for column groups if needed based on the column samples.
-// TODO: move this function to statistics/builder.go.
 func BuildExtendedStats(sctx sessionctx.Context,
- tableID int64, cols []*model.ColumnInfo, collectors []*statistics.SampleCollector) (*statistics.ExtendedStatsColl, error) {
+ tableID int64, cols []*model.ColumnInfo, collectors []*SampleCollector) (*ExtendedStatsColl, error) {
  const sql = "SELECT name, type, column_ids FROM mysql.stats_extended WHERE table_id = %? and status in (%?, %?)"
- rows, _, err := util.ExecRows(sctx, sql, tableID, statistics.ExtendedStatsAnalyzed, statistics.ExtendedStatsInited)
+
+ sqlExec, ok := sctx.(sqlexec.RestrictedSQLExecutor)
+ if !ok {
+ return nil, errors.Errorf("invalid sql executor")
+ }
+ rows, _, err := sqlExec.ExecRestrictedSQL(kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats), nil, sql, tableID, ExtendedStatsAnalyzed, ExtendedStatsInited)
  if err != nil {
  return nil, errors.Trace(err)
  }
  if len(rows) == 0 {
  return nil, nil
  }
- statsColl := statistics.NewExtendedStatsColl()
+ statsColl := NewExtendedStatsColl()
  for _, row := range rows {
  name := row.GetString(0)
- item := &statistics.ExtendedStatsItem{Tp: uint8(row.GetInt64(1))}
+ item := &ExtendedStatsItem{Tp: uint8(row.GetInt64(1))}
  colIDs := row.GetString(2)
  err := json.Unmarshal([]byte(colIDs), &item.ColIDs)
  if err != nil {
@@ -61,7 +65,7 @@ func BuildExtendedStats(sctx sessionctx.Context,
  return statsColl, nil
 }
 
-func fillExtendedStatsItemVals(sctx sessionctx.Context, item *statistics.ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*statistics.SampleCollector) *statistics.ExtendedStatsItem {
+func fillExtendedStatsItemVals(sctx sessionctx.Context, item *ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*SampleCollector) *ExtendedStatsItem {
  switch item.Tp {
  case ast.StatsTypeCardinality, ast.StatsTypeDependency:
  return nil
@@ -71,7 +75,7 @@ func fillExtendedStatsItemVals(sctx sessionctx.Context, item *statistics.Extende
  return nil
 }
 
-func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*statistics.SampleCollector) *statistics.ExtendedStatsItem {
+func fillExtStatsCorrVals(sctx sessionctx.Context, item *ExtendedStatsItem, cols []*model.ColumnInfo, collectors []*SampleCollector) *ExtendedStatsItem {
  colOffsets := make([]int, 0, 2)
  for _, id := range item.ColIDs {
  for i, col := range cols {
@@ -87,8 +91,8 @@ func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStat
  // samplesX and samplesY are in order of handle, i.e, their SampleItem.Ordinals are in order.
  samplesX := collectors[colOffsets[0]].Samples
  // We would modify Ordinal of samplesY, so we make a deep copy.
- samplesY := statistics.CopySampleItems(collectors[colOffsets[1]].Samples)
- sampleNum := mathutil.Min(len(samplesX), len(samplesY))
+ samplesY := CopySampleItems(collectors[colOffsets[1]].Samples)
+ sampleNum := min(len(samplesX), len(samplesY))
  if sampleNum == 1 {
  item.ScalarVals = 1
  return item
@@ -101,11 +105,11 @@ func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStat
  sc := sctx.GetSessionVars().StmtCtx
 
  var err error
- samplesX, err = statistics.SortSampleItems(sc, samplesX)
+ samplesX, err = SortSampleItems(sc, samplesX)
  if err != nil {
  return nil
  }
- samplesYInXOrder := make([]*statistics.SampleItem, 0, sampleNum)
+ samplesYInXOrder := make([]*SampleItem, 0, sampleNum)
  for i, itemX := range samplesX {
  if itemX.Ordinal >= len(samplesY) {
  continue
@@ -114,7 +118,7 @@ func fillExtStatsCorrVals(sctx sessionctx.Context, item *statistics.ExtendedStat
  itemY.Ordinal = i
  samplesYInXOrder = append(samplesYInXOrder, itemY)
  }
- samplesYInYOrder, err := statistics.SortSampleItems(sc, samplesYInXOrder)
+ samplesYInYOrder, err := SortSampleItems(sc, samplesYInXOrder)
  if err != nil {
  return nil
  }

pkg/statistics/handle/BUILD.bazel

@@ -16,7 +16,6 @@ go_library(
  "//pkg/infoschema",
  "//pkg/kv",
  "//pkg/metrics",
- "//pkg/parser/ast",
  "//pkg/parser/model",
  "//pkg/parser/mysql",
  "//pkg/parser/terror",
@@ -26,7 +25,6 @@ go_library(
  "//pkg/statistics",
  "//pkg/statistics/handle/autoanalyze",
  "//pkg/statistics/handle/cache",
- "//pkg/statistics/handle/extstats",
  "//pkg/statistics/handle/globalstats",
  "//pkg/statistics/handle/history",
  "//pkg/statistics/handle/lockstats",
@@ -50,21 +48,19 @@ go_test(
  timeout = "short",
  srcs = [
  "ddl_test.go",
- "gc_test.go",
  "handle_hist_test.go",
  "main_test.go",
  ],
  embed = [":handle"],
  flaky = True,
  race = "on",
- shard_count = 13,
+ shard_count = 8,
  deps = [
  "//pkg/config",
  "//pkg/parser/model",
  "//pkg/planner/cardinality",
  "//pkg/sessionctx",
  "//pkg/sessionctx/stmtctx",
- "//pkg/sessionctx/variable",
  "//pkg/testkit",
  "//pkg/testkit/testsetup",
  "//pkg/types",

pkg/statistics/handle/ddl.go

@@ -17,9 +17,7 @@ package handle
 import (
  "github.com/pingcap/tidb/pkg/ddl/util"
  "github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/sessionctx"
  "github.com/pingcap/tidb/pkg/sessionctx/variable"
- "github.com/pingcap/tidb/pkg/statistics/handle/globalstats"
 )
 
 // HandleDDLEvent begins to process a ddl task.
@@ -67,7 +65,7 @@ func (h *Handle) HandleDDLEvent(t *util.Event) error {
  return err
  }
  if variable.PartitionPruneMode(pruneMode) == variable.Dynamic && t.PartInfo != nil {
- if err := h.updateGlobalStats(t.TableInfo); err != nil {
+ if err := h.UpdateGlobalStats(t.TableInfo); err != nil {
  return err
  }
  }
@@ -105,14 +103,6 @@ func (h *Handle) HandleDDLEvent(t *util.Event) error {
  return nil
 }
 
-// updateGlobalStats will trigger the merge of global-stats when we drop table partition
-func (h *Handle) updateGlobalStats(tblInfo *model.TableInfo) error {
- // We need to merge the partition-level stats to global-stats when we drop table partition in dynamic mode.
- return h.callWithSCtx(func(sctx sessionctx.Context) error {
- return globalstats.UpdateGlobalStats(sctx, tblInfo, h.gpool, h.TableStatsFromStorage, h.TableInfoByID, h.callWithSCtx, h.SaveStatsToStorage)
- })
-}
-
 func (h *Handle) getInitStateTableIDs(tblInfo *model.TableInfo) (ids []int64, err error) {
  pi := tblInfo.GetPartitionInfo()
  if pi == nil {

pkg/statistics/handle/globalstats/BUILD.bazel

@@ -20,7 +20,6 @@ go_library(
  "//pkg/statistics",
  "//pkg/statistics/handle/storage",
  "//pkg/statistics/handle/util",
- "//pkg/table",
  "//pkg/types",
  "//pkg/util/hack",
  "//pkg/util/logutil",