From 4267d6e3a7e45241539b7f3116f8e4538bdcb753 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 22 Sep 2023 15:31:33 +0800 Subject: [PATCH 1/2] statistics: remove useless partition stats cache Signed-off-by: Weizhen Wang --- executor/analyze_global_stats.go | 8 -------- statistics/handle/ddl.go | 4 ++-- statistics/handle/globalstats/global_stats.go | 18 ++++-------------- statistics/handle/handle.go | 6 ++---- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/executor/analyze_global_stats.go b/executor/analyze_global_stats.go index 16c6a1fb901a1..50ac6dcf1ed43 100644 --- a/executor/analyze_global_stats.go +++ b/executor/analyze_global_stats.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/logutil" "go.uber.org/zap" - "golang.org/x/exp/maps" ) type globalStatsKey struct { @@ -54,10 +53,8 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob statsHandle := domain.GetDomain(e.Ctx()).StatsHandle() tableIDs := make(map[int64]struct{}, len(globalStatsTableIDs)) - tableAllPartitionStats := make(map[int64]*statistics.Table) for tableID := range globalStatsTableIDs { tableIDs[tableID] = struct{}{} - maps.Clear(tableAllPartitionStats) for globalStatsID, info := range globalStatsMap { if globalStatsID.tableID != tableID { @@ -86,7 +83,6 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob globalStatsID.tableID, info.isIndex == 1, info.histIDs, - tableAllPartitionStats, ) if err != nil { logutil.BgLogger().Warn("merge global stats failed", @@ -127,10 +123,6 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob FinishAnalyzeMergeJob(e.Ctx(), job, mergeStatsErr) } - - for _, value := range tableAllPartitionStats { - value.ReleaseAndPutToPool() - } } for tableID := range tableIDs { diff --git a/statistics/handle/ddl.go b/statistics/handle/ddl.go index 4b81bcfc6209e..736e737bf65a4 100644 --- a/statistics/handle/ddl.go +++ b/statistics/handle/ddl.go @@ -201,7 +201,7 @@ func (h *Handle) updateGlobalStats(tblInfo *model.TableInfo) error { opts[ast.AnalyzeOptNumBuckets] = uint64(globalColStatsBucketNum) } // Generate the new column global-stats - newColGlobalStats, err := h.mergePartitionStats2GlobalStats(opts, is, tblInfo, false, nil, nil) + newColGlobalStats, err := h.mergePartitionStats2GlobalStats(opts, is, tblInfo, false, nil) if err != nil { return err } @@ -240,7 +240,7 @@ func (h *Handle) updateGlobalStats(tblInfo *model.TableInfo) error { if globalIdxStatsBucketNum != 0 { opts[ast.AnalyzeOptNumBuckets] = uint64(globalIdxStatsBucketNum) } - newIndexGlobalStats, err := h.mergePartitionStats2GlobalStats(opts, is, tblInfo, true, []int64{idx.ID}, nil) + newIndexGlobalStats, err := h.mergePartitionStats2GlobalStats(opts, is, tblInfo, true, []int64{idx.ID}) if err != nil { return err } diff --git a/statistics/handle/globalstats/global_stats.go b/statistics/handle/globalstats/global_stats.go index a61d7c010d460..46c0fedfe5191 100644 --- a/statistics/handle/globalstats/global_stats.go +++ b/statistics/handle/globalstats/global_stats.go @@ -80,17 +80,11 @@ func MergePartitionStats2GlobalStats( globalTableInfo *model.TableInfo, isIndex bool, histIDs []int64, - allPartitionStats map[int64]*statistics.Table, getTableByPhysicalIDFn getTableByPhysicalIDFunc, loadTablePartitionStatsFn loadTablePartitionStatsFunc, ) (globalStats *GlobalStats, err error) { partitionNum := len(globalTableInfo.Partition.Definitions) - externalCache := false - if allPartitionStats == nil { - allPartitionStats = make(map[int64]*statistics.Table) - } else { - externalCache = true - } + allPartitionStats := make(map[int64]*statistics.Table) if len(histIDs) == 0 { for _, col := range globalTableInfo.Columns { // The virtual generated column stats can not be merged to the global stats. @@ -253,10 +247,8 @@ func MergePartitionStats2GlobalStats( globalStats.Fms[i].DestroyAndPutToPool() globalStats.Hg[i].NDV = globalStatsNDV } - if !externalCache { - for _, value := range allPartitionStats { - value.ReleaseAndPutToPool() - } + for _, value := range allPartitionStats { + value.ReleaseAndPutToPool() } return } @@ -270,7 +262,6 @@ func MergePartitionStats2GlobalStatsByTableID( physicalID int64, isIndex bool, histIDs []int64, - tablePartitionStats map[int64]*statistics.Table, getTableByPhysicalIDFn getTableByPhysicalIDFunc, loadTablePartitionStatsFn loadTablePartitionStatsFunc, ) (globalStats *GlobalStats, err error) { @@ -282,8 +273,7 @@ func MergePartitionStats2GlobalStatsByTableID( } globalTableInfo := globalTable.Meta() - globalStats, err = MergePartitionStats2GlobalStats(sc, gpool, opts, is, globalTableInfo, isIndex, histIDs, - tablePartitionStats, getTableByPhysicalIDFn, loadTablePartitionStatsFn) + globalStats, err = MergePartitionStats2GlobalStats(sc, gpool, opts, is, globalTableInfo, isIndex, histIDs, getTableByPhysicalIDFn, loadTablePartitionStatsFn) if err != nil { return } diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 255dda39dbbfd..856c5cadfb728 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -363,9 +363,8 @@ func (h *Handle) MergePartitionStats2GlobalStatsByTableID( physicalID int64, isIndex bool, histIDs []int64, - tablePartitionStats map[int64]*statistics.Table, ) (globalStats *globalstats.GlobalStats, err error) { - return globalstats.MergePartitionStats2GlobalStatsByTableID(sc, h.gpool, opts, is, physicalID, isIndex, histIDs, tablePartitionStats, h.getTableByPhysicalID, h.loadTablePartitionStats) + return globalstats.MergePartitionStats2GlobalStatsByTableID(sc, h.gpool, opts, is, physicalID, isIndex, histIDs, h.getTableByPhysicalID, h.loadTablePartitionStats) } func (h *Handle) loadTablePartitionStats(tableInfo *model.TableInfo, partitionDef *model.PartitionDefinition) (*statistics.Table, error) { @@ -390,7 +389,6 @@ func (h *Handle) mergePartitionStats2GlobalStats( globalTableInfo *model.TableInfo, isIndex bool, histIDs []int64, - allPartitionStats map[int64]*statistics.Table, ) (*globalstats.GlobalStats, error) { se, err := h.pool.Get() if err != nil { @@ -402,7 +400,7 @@ func (h *Handle) mergePartitionStats2GlobalStats( if err := UpdateSCtxVarsForStats(sc); err != nil { return nil, err } - return globalstats.MergePartitionStats2GlobalStats(sc, h.gpool, opts, is, globalTableInfo, isIndex, histIDs, allPartitionStats, h.getTableByPhysicalID, h.loadTablePartitionStats) + return globalstats.MergePartitionStats2GlobalStats(sc, h.gpool, opts, is, globalTableInfo, isIndex, histIDs, h.getTableByPhysicalID, h.loadTablePartitionStats) } func (h *Handle) getTableByPhysicalID(is infoschema.InfoSchema, physicalID int64) (table.Table, bool) { From db6c2ea24a07ee1d7c88ca6e6c13c119205a7eac Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 22 Sep 2023 15:47:21 +0800 Subject: [PATCH 2/2] update Signed-off-by: Weizhen Wang --- executor/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/executor/BUILD.bazel b/executor/BUILD.bazel index 57f64bfc4ea7b..cca4a87b7fc24 100644 --- a/executor/BUILD.bazel +++ b/executor/BUILD.bazel @@ -275,7 +275,6 @@ go_library( "@org_golang_google_grpc//credentials", "@org_golang_google_grpc//credentials/insecure", "@org_golang_google_grpc//status", - "@org_golang_x_exp//maps", "@org_golang_x_sync//errgroup", "@org_uber_go_atomic//:atomic", "@org_uber_go_zap//:zap",