Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

client: print more info when leader disconnect #7907

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
+29 −13
Open

client: print more info when leader disconnect #7907

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fdc5578
modify log
CabinfeverB Mar 12, 2024
3cb6185
set leader only changed
CabinfeverB Mar 12, 2024
79dc489
merge master
CabinfeverB Mar 12, 2024
1aaf5c8
add comment
CabinfeverB Mar 13, 2024
4de33c2
fix connection nil
CabinfeverB Mar 13, 2024
165a51c
address comment
CabinfeverB Mar 13, 2024
80ac87f
address comment
CabinfeverB Mar 13, 2024
6119ca3
Merge branch 'master' into client/modify_log_when_leader_disconnect
CabinfeverB Mar 13, 2024
6700333
address comment
CabinfeverB Mar 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/stretchr/testify v1.8.2
go.uber.org/atomic v1.10.0
go.uber.org/goleak v1.1.11
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.24.0
golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4
google.golang.org/grpc v1.59.0
Expand All @@ -33,7 +34,6 @@ require (
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.46.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.20.0 // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
Expand Down
32 changes: 21 additions & 11 deletions client/pd_service_discovery.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/tikv/pd/client/errs"
"github.com/tikv/pd/client/grpcutil"
"github.com/tikv/pd/client/retry"
"go.uber.org/multierr"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -1020,32 +1021,41 @@ func (c *pdServiceDiscovery) updateURLs(members []*pdpb.Member) {
log.Info("[pd] update member urls", zap.Strings("old-urls", oldURLs), zap.Strings("new-urls", urls))
}

func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) {
// switchLeader switches the leader of the PD cluster.
// Note: For current implementation, when initializing the client, the connection to leader should be established.
// Otherwise, the initialization will fail.
func (c *pdServiceDiscovery) switchLeader(addrs []string) (change bool, err error) {
// FIXME: How to safely compare leader urls? For now, only allows one client url.
addr := addrs[0]
oldLeader := c.getLeaderServiceClient()
if addr == oldLeader.GetAddress() && oldLeader.GetClientConn() != nil {
return false, nil
}

newConn, err := c.GetOrCreateGRPCConn(addr)
var newConn *grpc.ClientConn
newConn, err = c.GetOrCreateGRPCConn(addr)
// If gRPC connect is created successfully or leader is new, still saves.
if addr != oldLeader.GetAddress() || newConn != nil {
// Set PD leader and Global TSO Allocator (which is also the PD leader)
leaderClient := newPDServiceClient(addr, addr, c.tlsCfg, newConn, true)
c.leader.Store(leaderClient)
if addr != oldLeader.GetAddress() {
change = true
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might fail to connect to the leader?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but the leader does change and it will affect the follower proxy

log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader.GetAddress()))
}
if err == nil {
change = true
log.Info("[pd] successfully connected to leader", zap.String("leader", addr))
} else {
log.Warn("[pd] failed to connect leader", zap.String("leader", addr), errs.ZapError(err))
}
// Set PD leader and Global TSO Allocator (which is also the PD leader)
leaderClient := newPDServiceClient(addr, addr, c.tlsCfg, newConn, true)
c.leader.Store(leaderClient)
// Run callbacks
if c.tsoGlobalAllocLeaderUpdatedCb != nil {
if err := c.tsoGlobalAllocLeaderUpdatedCb(addr); err != nil {
return true, err
}
err = multierr.Append(err, c.tsoGlobalAllocLeaderUpdatedCb(addr))
}
for _, cb := range c.leaderSwitchedCbs {
cb()
}
log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader.GetAddress()))
return true, err
return
}

func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) (changed bool) {
Expand Down
Loading