Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

*: add defer to logs the panic reason and stack #6123

Merged
merged 5 commits into from
Mar 13, 2023
Merged

*: add defer to logs the panic reason and stack #6123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6088878
add log defer
HuSharp Mar 9, 2023
a157171
merge master
HuSharp Mar 13, 2023
702ca48
remove monitor log panic
HuSharp Mar 9, 2023
9504eec
Merge branch 'master' into add_log_defer
HuSharp Mar 13, 2023
22c0e31
Merge branch 'master' into add_log_defer
ti-chi-bot Mar 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pkg/cache/ttl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"time"

"github.com/pingcap/log"
"github.com/tikv/pd/pkg/utils/logutil"
"github.com/tikv/pd/pkg/utils/syncutil"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -142,6 +143,7 @@ func (c *ttlCache) Clear() {
}

func (c *ttlCache) doGC() {
defer logutil.LogPanic()
ticker := time.NewTicker(c.gcInterval)
defer ticker.Stop()

Expand Down
3 changes: 3 additions & 0 deletions pkg/election/lease.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/pingcap/log"
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/pkg/utils/etcdutil"
"github.com/tikv/pd/pkg/utils/logutil"
"github.com/tikv/pd/pkg/utils/typeutil"
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
Expand Down Expand Up @@ -129,6 +130,7 @@ func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-c
ch := make(chan time.Time)

go func() {
defer logutil.LogPanic()
ticker := time.NewTicker(interval)
defer ticker.Stop()

Expand All @@ -137,6 +139,7 @@ func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-c

for {
go func() {
defer logutil.LogPanic()
start := time.Now()
ctx1, cancel := context.WithTimeout(ctx, l.leaseTimeout)
defer cancel()
Expand Down
2 changes: 2 additions & 0 deletions pkg/gctuner/memory_limit_tuner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/pingcap/log"
util "github.com/tikv/pd/pkg/gogc"
"github.com/tikv/pd/pkg/memory"
"github.com/tikv/pd/pkg/utils/logutil"
atomicutil "go.uber.org/atomic"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -72,6 +73,7 @@ func (t *memoryLimitTuner) tuning() {
if float64(r.HeapInuse)*ratio > float64(setMemoryLimit(-1)) {
if t.nextGCTriggeredByMemoryLimit.Load() && t.waitingReset.CompareAndSwap(false, true) {
go func() {
defer logutil.LogPanic()
memory.MemoryLimitGCLast.Store(time.Now())
memory.MemoryLimitGCTotal.Add(1)
setMemoryLimit(t.calcMemoryLimit(fallbackPercentage))
Expand Down
2 changes: 2 additions & 0 deletions pkg/mcs/discovery/register.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"time"

"github.com/pingcap/log"
"github.com/tikv/pd/pkg/utils/logutil"
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -70,6 +71,7 @@ func (sr *ServiceRegister) Register() error {
return fmt.Errorf("keepalive failed: %v", err)
}
go func() {
defer logutil.LogPanic()
for {
select {
case <-sr.ctx.Done():
Expand Down
7 changes: 6 additions & 1 deletion pkg/mcs/resource_manager/server/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
bs "github.com/tikv/pd/pkg/basicserver"
"github.com/tikv/pd/pkg/storage/endpoint"
"github.com/tikv/pd/pkg/storage/kv"
"github.com/tikv/pd/pkg/utils/logutil"
"go.uber.org/zap"
)

Expand Down Expand Up @@ -121,7 +122,10 @@ func (m *Manager) Init(ctx context.Context) {
m.storage.LoadResourceGroupStates(tokenHandler)
// Start the background metrics flusher.
go m.backgroundMetricsFlush(ctx)
go m.persistLoop(ctx)
go func() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the difference between backgroundMetricsFlush and persistLoop?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

persistLoop function requires a look at the code to know that it is a goroutine and is not used anywhere else.
To avoid adding logpanic to confuse people, it was changed to this

defer logutil.LogPanic()
m.persistLoop(ctx)
}()
log.Info("resource group manager finishes initialization")
}

Expand Down Expand Up @@ -249,6 +253,7 @@ func (m *Manager) persistResourceGroupRunningState() {

// Receive the consumption and flush it to the metrics.
func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
defer logutil.LogPanic()
ticker := time.NewTicker(metricsCleanupInterval)
defer ticker.Stop()
for {
Expand Down
4 changes: 4 additions & 0 deletions pkg/mcs/resource_manager/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ func (s *Server) initClient() error {
}

func (s *Server) startGRPCServer(l net.Listener) {
defer logutil.LogPanic()
defer s.serverLoopWg.Done()

gs := grpc.NewServer()
Expand All @@ -282,6 +283,7 @@ func (s *Server) startGRPCServer(l net.Listener) {
// it doesn't happen in a reasonable amount of time.
done := make(chan struct{})
go func() {
defer logutil.LogPanic()
log.Info("try to gracefully stop the server now")
gs.GracefulStop()
close(done)
Expand All @@ -300,6 +302,7 @@ func (s *Server) startGRPCServer(l net.Listener) {
}

func (s *Server) startHTTPServer(l net.Listener) {
defer logutil.LogPanic()
defer s.serverLoopWg.Done()

handler, _ := SetUpRestHandler(s.service)
Expand All @@ -326,6 +329,7 @@ func (s *Server) startHTTPServer(l net.Listener) {
}

func (s *Server) startGRPCAndHTTPServers(l net.Listener) {
defer logutil.LogPanic()
defer s.serverLoopWg.Done()

mux := cmux.New(l)
Expand Down
3 changes: 3 additions & 0 deletions pkg/mcs/tso/server/grpc_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/tikv/pd/pkg/mcs/registry"
"github.com/tikv/pd/pkg/utils/apiutil"
"github.com/tikv/pd/pkg/utils/grpcutil"
"github.com/tikv/pd/pkg/utils/logutil"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -191,6 +192,7 @@ func (s *Service) dispatchTSORequest(ctx context.Context, request *tsoRequest, f
}

func (s *Service) handleDispatcher(ctx context.Context, forwardedHost string, tsoRequestCh <-chan *tsoRequest, tsDeadlineCh chan<- deadline, doneCh <-chan struct{}, errCh chan<- error) {
defer logutil.LogPanic()
dispatcherCtx, ctxCancel := context.WithCancel(ctx)
defer ctxCancel()
defer s.tsoDispatcher.Delete(forwardedHost)
Expand Down Expand Up @@ -330,6 +332,7 @@ type deadline struct {
}

func watchTSDeadline(ctx context.Context, tsDeadlineCh <-chan deadline) {
defer logutil.LogPanic()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
for {
Expand Down
5 changes: 5 additions & 0 deletions pkg/mcs/tso/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,7 @@ func (s *Server) SetExternalTS(externalTS uint64) error {
}

func checkStream(streamCtx context.Context, cancel context.CancelFunc, done chan struct{}) {
defer logutil.LogPanic()
select {
case <-done:
return
Expand Down Expand Up @@ -479,6 +480,7 @@ func (s *Server) initClient() error {
}

func (s *Server) startGRPCServer(l net.Listener) {
defer logutil.LogPanic()
defer s.serverLoopWg.Done()

gs := grpc.NewServer()
Expand All @@ -491,6 +493,7 @@ func (s *Server) startGRPCServer(l net.Listener) {
// it doesn't happen in a reasonable amount of time.
done := make(chan struct{})
go func() {
defer logutil.LogPanic()
log.Info("try to gracefully stop the server now")
gs.GracefulStop()
close(done)
Expand All @@ -510,6 +513,7 @@ func (s *Server) startGRPCServer(l net.Listener) {
}

func (s *Server) startHTTPServer(l net.Listener) {
defer logutil.LogPanic()
defer s.serverLoopWg.Done()

handler, _ := SetUpRestHandler(s.service)
Expand All @@ -536,6 +540,7 @@ func (s *Server) startHTTPServer(l net.Listener) {
}

func (s *Server) startGRPCAndHTTPServers(l net.Listener) {
defer logutil.LogPanic()
defer s.serverLoopWg.Done()

mux := cmux.New(l)
Expand Down
5 changes: 1 addition & 4 deletions pkg/schedule/region_scatterer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,10 +407,7 @@ func allowLeader(fit *placement.RegionFit, peer *metapb.Peer) bool {
return false
}
peerFit := fit.GetRuleFit(peer.GetId())
if peerFit == nil || peerFit.Rule == nil {
return false
}
if peerFit.Rule.IsWitness {
if peerFit == nil || peerFit.Rule == nil || peerFit.Rule.IsWitness {
return false
}
switch peerFit.Rule.Role {
Expand Down
2 changes: 2 additions & 0 deletions pkg/systimemon/systimemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ import (

"github.com/pingcap/log"
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/pkg/utils/logutil"
"go.uber.org/zap"
)

// StartMonitor calls systimeErrHandler if system time jump backward.
func StartMonitor(ctx context.Context, now func() time.Time, systimeErrHandler func()) {
defer logutil.LogPanic()
log.Info("start system time monitor")
tick := time.NewTicker(100 * time.Millisecond)
defer tick.Stop()
Expand Down
4 changes: 4 additions & 0 deletions pkg/tso/allocator_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/tikv/pd/pkg/storage/kv"
"github.com/tikv/pd/pkg/utils/etcdutil"
"github.com/tikv/pd/pkg/utils/grpcutil"
"github.com/tikv/pd/pkg/utils/logutil"
"github.com/tikv/pd/pkg/utils/syncutil"
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
Expand Down Expand Up @@ -411,6 +412,7 @@ func (am *AllocatorManager) getLocalTSOAllocatorPath() string {

// similar logic with leaderLoop in server/server.go
func (am *AllocatorManager) allocatorLeaderLoop(ctx context.Context, allocator *LocalTSOAllocator) {
defer logutil.LogPanic()
defer log.Info("server is closed, return local tso allocator leader loop",
zap.String("dc-location", allocator.GetDCLocation()),
zap.String("local-tso-allocator-name", am.member.Member().Name))
Expand Down Expand Up @@ -662,6 +664,7 @@ func (am *AllocatorManager) allocatorUpdater() {

// updateAllocator is used to update the allocator in the group.
func (am *AllocatorManager) updateAllocator(ag *allocatorGroup) {
defer logutil.LogPanic()
defer am.wg.Done()
select {
case <-ag.ctx.Done():
Expand Down Expand Up @@ -712,6 +715,7 @@ func (am *AllocatorManager) allocatorPatroller(serverCtx context.Context) {
// ClusterDCLocationChecker collects all dc-locations of a cluster, computes some related info
// and stores them into the DCLocationInfo, then finally writes them into am.mu.clusterDCLocations.
func (am *AllocatorManager) ClusterDCLocationChecker() {
defer logutil.LogPanic()
// Wait for the PD leader to be elected out.
if am.member.GetLeader() == nil {
return
Expand Down
2 changes: 2 additions & 0 deletions pkg/tso/global_allocator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/tikv/pd/pkg/election"
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/pkg/slice"
"github.com/tikv/pd/pkg/utils/logutil"
"github.com/tikv/pd/pkg/utils/tsoutil"
"github.com/tikv/pd/pkg/utils/typeutil"
"go.uber.org/zap"
Expand Down Expand Up @@ -340,6 +341,7 @@ func (gta *GlobalTSOAllocator) SyncMaxTS(
// Send SyncMaxTSRequest to all allocator leaders concurrently.
wg.Add(1)
go func(ctx context.Context, conn *grpc.ClientConn, respCh chan<- *syncResp) {
defer logutil.LogPanic()
defer wg.Done()
syncMaxTSResp := &syncResp{}
syncCtx, cancel := context.WithTimeout(ctx, rpcTimeout)
Expand Down
2 changes: 2 additions & 0 deletions server/region_syncer/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/tikv/pd/pkg/errs"
"github.com/tikv/pd/pkg/storage"
"github.com/tikv/pd/pkg/utils/grpcutil"
"github.com/tikv/pd/pkg/utils/logutil"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
Expand Down Expand Up @@ -117,6 +118,7 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) {
ctx := s.mu.clientCtx

go func() {
defer logutil.LogPanic()
defer s.wg.Done()
// used to load region from kv storage to cache storage.
bc := s.server.GetBasicCluster()
Expand Down