From 5f6fdd1dd4854cf04d0b1956eb8b50f26c55d92d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 25 Sep 2023 00:37:37 +0800 Subject: [PATCH 1/4] *: remove useless cache in the MergePartitionStats2GlobalStats Signed-off-by: Weizhen Wang --- statistics/handle/globalstats/global_stats.go | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/statistics/handle/globalstats/global_stats.go b/statistics/handle/globalstats/global_stats.go index 63aba227e66c0..14d17729e93c8 100644 --- a/statistics/handle/globalstats/global_stats.go +++ b/statistics/handle/globalstats/global_stats.go @@ -84,7 +84,6 @@ func MergePartitionStats2GlobalStats( loadTablePartitionStatsFn loadTablePartitionStatsFunc, ) (globalStats *GlobalStats, err error) { partitionNum := len(globalTableInfo.Partition.Definitions) - allPartitionStats := make(map[int64]*statistics.Table) if len(histIDs) == 0 { for _, col := range globalTableInfo.Columns { // The virtual generated column stats can not be merged to the global stats. @@ -122,24 +121,18 @@ func MergePartitionStats2GlobalStats( err = errors.Errorf("unknown physical ID %d in stats meta table, maybe it has been dropped", partitionID) return } - tableInfo := partitionTable.Meta() var partitionStats *statistics.Table - - partitionStats, ok = allPartitionStats[partitionID] // If pre-load partition stats isn't provided, then we load partition stats directly and set it into allPartitionStats - if !ok { - var err1 error - partitionStats, err1 = loadTablePartitionStatsFn(tableInfo, &def) - if err1 != nil { - if skipMissingPartitionStats && types.ErrPartitionStatsMissing.Equal(err) { - globalStats.MissingPartitionStats = append(globalStats.MissingPartitionStats, fmt.Sprintf("partition `%s`", def.Name.L)) - continue - } - err = err1 - return + var err1 error + partitionStats, err1 = loadTablePartitionStatsFn(tableInfo, &def) + if err1 != nil { + if skipMissingPartitionStats && types.ErrPartitionStatsMissing.Equal(err) { + globalStats.MissingPartitionStats = append(globalStats.MissingPartitionStats, fmt.Sprintf("partition `%s`", def.Name.L)) + continue } - allPartitionStats[partitionID] = partitionStats + err = err1 + return } for i := 0; i < globalStats.Num; i++ { @@ -189,6 +182,7 @@ func MergePartitionStats2GlobalStats( allFms[i] = append(allFms[i], fms) } } + partitionStats.ReleaseAndPutToPool() } // After collect all the statistics from the partition-level stats, @@ -247,9 +241,6 @@ func MergePartitionStats2GlobalStats( globalStats.Fms[i].DestroyAndPutToPool() globalStats.Hg[i].NDV = globalStatsNDV } - for _, value := range allPartitionStats { - value.ReleaseAndPutToPool() - } return } From 0a217fff2559ed1338ededf432d728353e53cf5b Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 25 Sep 2023 13:45:25 +0800 Subject: [PATCH 2/4] update Signed-off-by: Weizhen Wang --- statistics/handle/globalstats/global_stats.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/statistics/handle/globalstats/global_stats.go b/statistics/handle/globalstats/global_stats.go index 14d17729e93c8..233013d185d50 100644 --- a/statistics/handle/globalstats/global_stats.go +++ b/statistics/handle/globalstats/global_stats.go @@ -123,9 +123,7 @@ func MergePartitionStats2GlobalStats( } tableInfo := partitionTable.Meta() var partitionStats *statistics.Table - // If pre-load partition stats isn't provided, then we load partition stats directly and set it into allPartitionStats - var err1 error - partitionStats, err1 = loadTablePartitionStatsFn(tableInfo, &def) + partitionStats, err1 := loadTablePartitionStatsFn(tableInfo, &def) if err1 != nil { if skipMissingPartitionStats && types.ErrPartitionStatsMissing.Equal(err) { globalStats.MissingPartitionStats = append(globalStats.MissingPartitionStats, fmt.Sprintf("partition `%s`", def.Name.L)) From b5638c4c22883d4a59f7a96b857104c9b4500445 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 25 Sep 2023 13:46:49 +0800 Subject: [PATCH 3/4] update Signed-off-by: Weizhen Wang --- statistics/handle/globalstats/global_stats.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/statistics/handle/globalstats/global_stats.go b/statistics/handle/globalstats/global_stats.go index 233013d185d50..eddf445f2b21b 100644 --- a/statistics/handle/globalstats/global_stats.go +++ b/statistics/handle/globalstats/global_stats.go @@ -134,6 +134,8 @@ func MergePartitionStats2GlobalStats( } for i := 0; i < globalStats.Num; i++ { + // GetStatsInfo will return the copy of the statsInfo, so we don't need to worry about the memory problem. + // partitionStats will be released after the for loop. hg, cms, topN, fms, analyzed := partitionStats.GetStatsInfo(histIDs[i], isIndex) skipPartition := false if !analyzed { From 07548644a42a111f6005ac7c5e817e1d02e4bab6 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 25 Sep 2023 15:11:01 +0800 Subject: [PATCH 4/4] update Signed-off-by: Weizhen Wang --- statistics/handle/globalstats/global_stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics/handle/globalstats/global_stats.go b/statistics/handle/globalstats/global_stats.go index eddf445f2b21b..7764d461cff92 100644 --- a/statistics/handle/globalstats/global_stats.go +++ b/statistics/handle/globalstats/global_stats.go @@ -134,7 +134,7 @@ func MergePartitionStats2GlobalStats( } for i := 0; i < globalStats.Num; i++ { - // GetStatsInfo will return the copy of the statsInfo, so we don't need to worry about the memory problem. + // GetStatsInfo will return the copy of the statsInfo, so we don't need to worry about the data race. // partitionStats will be released after the for loop. hg, cms, topN, fms, analyzed := partitionStats.GetStatsInfo(histIDs[i], isIndex) skipPartition := false