From 79ab2b92a046bba03f957c4bf421eb9fb55bc053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E5=B2=9A?= <36239017+YuJuncen@users.noreply.github.com> Date: Mon, 26 Dec 2022 22:38:15 +0800 Subject: [PATCH 1/3] restore: Fix prealloc (#40176) close pingcap/tidb#40177 --- br/pkg/restore/db.go | 2 +- br/pkg/restore/prealloc_table_id/BUILD.bazel | 5 ++- br/pkg/restore/prealloc_table_id/alloc.go | 23 +++++++++++ .../restore/prealloc_table_id/alloc_test.go | 38 ++++++++++++++++--- 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/br/pkg/restore/db.go b/br/pkg/restore/db.go index ae62162c3e890..132e3294f1617 100644 --- a/br/pkg/restore/db.go +++ b/br/pkg/restore/db.go @@ -284,7 +284,7 @@ func (db *DB) tableIDAllocFilter() ddl.AllocTableIDIf { if db.preallocedIDs == nil { return true } - prealloced := db.preallocedIDs.Prealloced(ti.ID) + prealloced := db.preallocedIDs.PreallocedFor(ti) if prealloced { log.Info("reusing table ID", zap.Stringer("table", ti.Name)) } diff --git a/br/pkg/restore/prealloc_table_id/BUILD.bazel b/br/pkg/restore/prealloc_table_id/BUILD.bazel index 8ce80b039178a..cfdb0432fd446 100644 --- a/br/pkg/restore/prealloc_table_id/BUILD.bazel +++ b/br/pkg/restore/prealloc_table_id/BUILD.bazel @@ -5,7 +5,10 @@ go_library( srcs = ["alloc.go"], importpath = "github.com/pingcap/tidb/br/pkg/restore/prealloc_table_id", visibility = ["//visibility:public"], - deps = ["//br/pkg/metautil"], + deps = [ + "//br/pkg/metautil", + "//parser/model", + ], ) go_test( diff --git a/br/pkg/restore/prealloc_table_id/alloc.go b/br/pkg/restore/prealloc_table_id/alloc.go index 9232ed84a8fc8..8554de5e9891b 100644 --- a/br/pkg/restore/prealloc_table_id/alloc.go +++ b/br/pkg/restore/prealloc_table_id/alloc.go @@ -7,6 +7,7 @@ import ( "math" "github.com/pingcap/tidb/br/pkg/metautil" + "github.com/pingcap/tidb/parser/model" ) const ( @@ -48,6 +49,14 @@ func New(tables []*metautil.Table) *PreallocIDs { if t.Info.ID > max && t.Info.ID < insaneTableIDThreshold { max = t.Info.ID } + + if t.Info.Partition != nil && t.Info.Partition.Definitions != nil { + for _, part := range t.Info.Partition.Definitions { + if part.ID > max && part.ID < insaneTableIDThreshold { + max = part.ID + } + } + } } return &PreallocIDs{ end: max + 1, @@ -86,3 +95,17 @@ func (p *PreallocIDs) Alloc(m Allocator) error { func (p *PreallocIDs) Prealloced(tid int64) bool { return p.allocedFrom <= tid && tid < p.end } + +func (p *PreallocIDs) PreallocedFor(ti *model.TableInfo) bool { + if !p.Prealloced(ti.ID) { + return false + } + if ti.Partition != nil && ti.Partition.Definitions != nil { + for _, part := range ti.Partition.Definitions { + if !p.Prealloced(part.ID) { + return false + } + } + } + return true +} diff --git a/br/pkg/restore/prealloc_table_id/alloc_test.go b/br/pkg/restore/prealloc_table_id/alloc_test.go index 8cf6b95fb070e..c1c3f018a2de8 100644 --- a/br/pkg/restore/prealloc_table_id/alloc_test.go +++ b/br/pkg/restore/prealloc_table_id/alloc_test.go @@ -27,6 +27,7 @@ func (t *testAllocator) AdvanceGlobalIDs(n int) (int64, error) { func TestAllocator(t *testing.T) { type Case struct { tableIDs []int64 + partitions map[int64][]int64 hasAllocatedTo int64 successfullyAllocated []int64 shouldAllocatedTo int64 @@ -57,16 +58,41 @@ func TestAllocator(t *testing.T) { successfullyAllocated: []int64{5, 6}, shouldAllocatedTo: 7, }, + { + tableIDs: []int64{1, 2, 5, 6, 7}, + hasAllocatedTo: 6, + successfullyAllocated: []int64{6, 7}, + shouldAllocatedTo: 13, + partitions: map[int64][]int64{ + 7: {8, 9, 10, 11, 12}, + }, + }, + { + tableIDs: []int64{1, 2, 5, 6, 7, 13}, + hasAllocatedTo: 9, + successfullyAllocated: []int64{13}, + shouldAllocatedTo: 14, + partitions: map[int64][]int64{ + 7: {8, 9, 10, 11, 12}, + }, + }, } run := func(t *testing.T, c Case) { tables := make([]*metautil.Table, 0, len(c.tableIDs)) for _, id := range c.tableIDs { - tables = append(tables, &metautil.Table{ + table := metautil.Table{ Info: &model.TableInfo{ - ID: id, + ID: id, + Partition: &model.PartitionInfo{}, }, - }) + } + if c.partitions != nil { + for _, part := range c.partitions[id] { + table.Info.Partition.Definitions = append(table.Info.Partition.Definitions, model.PartitionDefinition{ID: part}) + } + } + tables = append(tables, &table) } ids := prealloctableid.New(tables) @@ -74,9 +100,9 @@ func TestAllocator(t *testing.T) { require.NoError(t, ids.Alloc(&allocator)) allocated := make([]int64, 0, len(c.successfullyAllocated)) - for _, t := range c.tableIDs { - if ids.Prealloced(t) { - allocated = append(allocated, t) + for _, t := range tables { + if ids.PreallocedFor(t.Info) { + allocated = append(allocated, t.Info.ID) } } require.ElementsMatch(t, allocated, c.successfullyAllocated) From ab01065a8541cab6eae0e6a0e6297d6047d7feb7 Mon Sep 17 00:00:00 2001 From: CbcWestwolf <1004626265@qq.com> Date: Tue, 27 Dec 2022 10:56:17 +0800 Subject: [PATCH 2/3] metrics: remove Keep Alive OPM metric (#40142) close pingcap/tidb#40143 --- metrics/metrics.go | 1 - metrics/server.go | 8 -------- tidb-server/main.go | 11 +---------- util/systimemon/systime_mon.go | 9 +-------- util/systimemon/systime_mon_test.go | 2 +- 5 files changed, 3 insertions(+), 28 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index f609caaa124cd..8f303ba58180e 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -124,7 +124,6 @@ func RegisterMetrics() { prometheus.MustRegister(SyncLoadHistogram) prometheus.MustRegister(ReadStatsHistogram) prometheus.MustRegister(JobsGauge) - prometheus.MustRegister(KeepAliveCounter) prometheus.MustRegister(LoadPrivilegeCounter) prometheus.MustRegister(InfoCacheCounters) prometheus.MustRegister(LoadSchemaCounter) diff --git a/metrics/server.go b/metrics/server.go index 6bd34c4a8b903..830e03b28e986 100644 --- a/metrics/server.go +++ b/metrics/server.go @@ -120,14 +120,6 @@ var ( Help: "Counter of system time jumps backward.", }) - KeepAliveCounter = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "monitor", - Name: "keep_alive_total", - Help: "Counter of TiDB keep alive.", - }) - PlanCacheCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "tidb", diff --git a/tidb-server/main.go b/tidb-server/main.go index 8605c7d0c3aaa..0e42c059e6c05 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -788,16 +788,7 @@ func setupMetrics() { systimeErrHandler := func() { metrics.TimeJumpBackCounter.Inc() } - callBackCount := 0 - successCallBack := func() { - callBackCount++ - // It is callback by monitor per second, we increase metrics.KeepAliveCounter per 5s. - if callBackCount >= 5 { - callBackCount = 0 - metrics.KeepAliveCounter.Inc() - } - } - go systimemon.StartMonitor(time.Now, systimeErrHandler, successCallBack) + go systimemon.StartMonitor(time.Now, systimeErrHandler) pushMetric(cfg.Status.MetricsAddr, time.Duration(cfg.Status.MetricsInterval)*time.Second) } diff --git a/util/systimemon/systime_mon.go b/util/systimemon/systime_mon.go index 67b23ed894b5b..26819a4159776 100644 --- a/util/systimemon/systime_mon.go +++ b/util/systimemon/systime_mon.go @@ -22,11 +22,10 @@ import ( ) // StartMonitor calls systimeErrHandler if system time jump backward. -func StartMonitor(now func() time.Time, systimeErrHandler func(), successCallback func()) { +func StartMonitor(now func() time.Time, systimeErrHandler func()) { logutil.BgLogger().Info("start system time monitor") tick := time.NewTicker(100 * time.Millisecond) defer tick.Stop() - tickCount := 0 for { last := now().UnixNano() <-tick.C @@ -34,11 +33,5 @@ func StartMonitor(now func() time.Time, systimeErrHandler func(), successCallbac logutil.BgLogger().Error("system time jump backward", zap.Int64("last", last)) systimeErrHandler() } - // call successCallback per second. - tickCount++ - if tickCount >= 10 { - tickCount = 0 - successCallback() - } } } diff --git a/util/systimemon/systime_mon_test.go b/util/systimemon/systime_mon_test.go index f05098b63db7e..6964a9ac3d70d 100644 --- a/util/systimemon/systime_mon_test.go +++ b/util/systimemon/systime_mon_test.go @@ -35,7 +35,7 @@ func TestSystimeMonitor(t *testing.T) { return time.Now().Add(-2 * time.Second) }, func() { errTriggered.Store(true) - }, func() {}) + }) require.Eventually(t, errTriggered.Load, time.Second, 10*time.Millisecond) } From 3c8f11da6a26839c5b0895fbbdf43412047b9e49 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 27 Dec 2022 11:32:17 +0800 Subject: [PATCH 3/3] util: fix data race in the cpu test (#40154) close pingcap/tidb#40125 --- util/cpu/BUILD.bazel | 1 + util/cpu/cpu_test.go | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/cpu/BUILD.bazel b/util/cpu/BUILD.bazel index 08893520caaa0..b831ba544947b 100644 --- a/util/cpu/BUILD.bazel +++ b/util/cpu/BUILD.bazel @@ -21,5 +21,6 @@ go_test( srcs = ["cpu_test.go"], embed = [":cpu"], flaky = True, + race = "on", deps = ["@com_github_stretchr_testify//require"], ) diff --git a/util/cpu/cpu_test.go b/util/cpu/cpu_test.go index cd330a11e5196..a191227b72f78 100644 --- a/util/cpu/cpu_test.go +++ b/util/cpu/cpu_test.go @@ -24,8 +24,7 @@ import ( ) func TestCPUValue(t *testing.T) { - Observer := NewCPUObserver() - Observer.Start() + observer := NewCPUObserver() exit := make(chan struct{}) var wg sync.WaitGroup for i := 0; i < 10; i++ { @@ -42,11 +41,11 @@ func TestCPUValue(t *testing.T) { } }() } - Observer.Start() + observer.Start() time.Sleep(5 * time.Second) require.GreaterOrEqual(t, GetCPUUsage(), 0.0) require.Less(t, GetCPUUsage(), 1.0) - Observer.Stop() + observer.Stop() close(exit) wg.Wait() }