From 3c9ed9cfe06720a57e8559c6409991698e241303 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 15:02:08 +0800 Subject: [PATCH 1/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/table.go | 42 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index a7be276c83ca1..46590577c33e6 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -653,12 +653,12 @@ func (t *Table) IndexIsLoadNeeded(id int64) (*Index, bool) { return idx, false } -type neededStatsMap struct { +type neededStatsInternalMap struct { items map[model.TableItemID]struct{} m sync.RWMutex } -func (n *neededStatsMap) AllItems() []model.TableItemID { +func (n *neededStatsInternalMap) AllItems() []model.TableItemID { n.m.RLock() keys := make([]model.TableItemID, 0, len(n.items)) for key := range n.items { @@ -668,24 +668,56 @@ func (n *neededStatsMap) AllItems() []model.TableItemID { return keys } -func (n *neededStatsMap) Insert(col model.TableItemID) { +func (n *neededStatsInternalMap) Insert(col model.TableItemID) { n.m.Lock() n.items[col] = struct{}{} n.m.Unlock() } -func (n *neededStatsMap) Delete(col model.TableItemID) { +func (n *neededStatsInternalMap) Delete(col model.TableItemID) { n.m.Lock() delete(n.items, col) n.m.Unlock() } -func (n *neededStatsMap) Length() int { +func (n *neededStatsInternalMap) Length() int { n.m.RLock() defer n.m.RUnlock() return len(n.items) } +const shardCnt = 64 + +type neededStatsMap struct { + items [shardCnt]neededStatsInternalMap + m sync.RWMutex +} + +func (n *neededStatsMap) AllItems() []model.TableItemID { + var result []model.TableItemID + for i := 0; i < 64; i++ { + keys := n.items[i].AllItems() + result = append(result, keys...) + } + return result +} + +func (n *neededStatsMap) Insert(col model.TableItemID) { + n.items[col.ID%shardCnt].Insert(col) +} + +func (n *neededStatsMap) Delete(col model.TableItemID) { + n.items[col.ID%shardCnt].Delete(col) +} + +func (n *neededStatsMap) Length() int { + var result int + for i := 0; i < 64; i++ { + result += n.items[i].Length() + } + return result +} + // RatioOfPseudoEstimate means if modifyCount / statsTblCount is greater than this ratio, we think the stats is invalid // and use pseudo estimation. var RatioOfPseudoEstimate = atomic.NewFloat64(0.7) From e80e06f07464c21658665fc305c9a5d242e01389 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 15:02:50 +0800 Subject: [PATCH 2/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index 46590577c33e6..7b69b876dc3dd 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -695,7 +695,7 @@ type neededStatsMap struct { func (n *neededStatsMap) AllItems() []model.TableItemID { var result []model.TableItemID - for i := 0; i < 64; i++ { + for i := 0; i < shardCnt; i++ { keys := n.items[i].AllItems() result = append(result, keys...) } @@ -712,7 +712,7 @@ func (n *neededStatsMap) Delete(col model.TableItemID) { func (n *neededStatsMap) Length() int { var result int - for i := 0; i < 64; i++ { + for i := 0; i < shardCnt; i++ { result += n.items[i].Length() } return result From 345490e4b102cf03dd407c8a6dccf111d5a690ca Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 15:13:46 +0800 Subject: [PATCH 3/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/column.go | 2 +- pkg/statistics/table.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/statistics/column.go b/pkg/statistics/column.go index 224de21cd16a6..4ad4b444451b3 100644 --- a/pkg/statistics/column.go +++ b/pkg/statistics/column.go @@ -134,7 +134,7 @@ func (c *Column) MemoryUsage() CacheItemMemoryUsage { // HistogramNeededItems stores the columns/indices whose Histograms need to be loaded from physical kv layer. // Currently, we only load index/pk's Histogram from kv automatically. Columns' are loaded by needs. -var HistogramNeededItems = neededStatsMap{items: map[model.TableItemID]struct{}{}} +var HistogramNeededItems = newNeededStatsMap() // ColumnStatsIsInvalid checks if this column is invalid. // If this column has histogram but not loaded yet, diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index 7b69b876dc3dd..d17f0b0cb0c64 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -693,6 +693,16 @@ type neededStatsMap struct { m sync.RWMutex } +func newNeededStatsMap() neededStatsMap { + result := neededStatsMap{} + for i := 0; i < shardCnt; i++ { + result.items[i] = neededStatsInternalMap{ + items: make(map[model.TableItemID]struct{}), + } + } + return result +} + func (n *neededStatsMap) AllItems() []model.TableItemID { var result []model.TableItemID for i := 0; i < shardCnt; i++ { From 65ffaa17cda7bae759e6421bcec07593a7776498 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 15:25:16 +0800 Subject: [PATCH 4/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index d17f0b0cb0c64..e75e66d84d2be 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -693,14 +693,14 @@ type neededStatsMap struct { m sync.RWMutex } -func newNeededStatsMap() neededStatsMap { +func newNeededStatsMap() *neededStatsMap { result := neededStatsMap{} for i := 0; i < shardCnt; i++ { result.items[i] = neededStatsInternalMap{ items: make(map[model.TableItemID]struct{}), } } - return result + return &result } func (n *neededStatsMap) AllItems() []model.TableItemID { From af3f7fe0e24191acd732a47e58fb4e0c1f590b7e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 15:25:50 +0800 Subject: [PATCH 5/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index e75e66d84d2be..1587f66197967 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -686,7 +686,7 @@ func (n *neededStatsInternalMap) Length() int { return len(n.items) } -const shardCnt = 64 +const shardCnt = 128 type neededStatsMap struct { items [shardCnt]neededStatsInternalMap From 4bd4b706f72fa7b9693b91fd8afa1386ac3c32a5 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 15:35:39 +0800 Subject: [PATCH 6/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/table.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index 1587f66197967..42ff8606356c8 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -690,7 +690,6 @@ const shardCnt = 128 type neededStatsMap struct { items [shardCnt]neededStatsInternalMap - m sync.RWMutex } func newNeededStatsMap() *neededStatsMap { From cd71419b96dea4eb283399c0d25fd3e9d2becc6c Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 16:19:11 +0800 Subject: [PATCH 7/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/table.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index 42ff8606356c8..ce917919cbd35 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -692,6 +692,14 @@ type neededStatsMap struct { items [shardCnt]neededStatsInternalMap } +func getIdx(tbl model.TableItemID) int64 { + result := tbl.ID % shardCnt + if result < 0 { + return -result + } + return result +} + func newNeededStatsMap() *neededStatsMap { result := neededStatsMap{} for i := 0; i < shardCnt; i++ { @@ -712,11 +720,11 @@ func (n *neededStatsMap) AllItems() []model.TableItemID { } func (n *neededStatsMap) Insert(col model.TableItemID) { - n.items[col.ID%shardCnt].Insert(col) + n.items[getIdx(col)].Insert(col) } func (n *neededStatsMap) Delete(col model.TableItemID) { - n.items[col.ID%shardCnt].Delete(col) + n.items[getIdx(col)].Delete(col) } func (n *neededStatsMap) Length() int { From 2a701d49d6fc394952cc426586eae4fa744f4025 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 28 Mar 2024 17:29:38 +0800 Subject: [PATCH 8/8] statistics: shard needsStatsMap Signed-off-by: Weizhen Wang --- pkg/statistics/table.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index ce917919cbd35..b3adb3597e4d4 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -693,11 +693,13 @@ type neededStatsMap struct { } func getIdx(tbl model.TableItemID) int64 { - result := tbl.ID % shardCnt - if result < 0 { - return -result + var id int64 + if tbl.ID < 0 { + id = -tbl.ID + } else { + id = tbl.ID } - return result + return id % shardCnt } func newNeededStatsMap() *neededStatsMap {