fix: automatic fix master-slave replication relationship after master…

… or slave service restarted (OpenAtomFoundation#2373, OpenAtomFoundation#2038, OpenAtomFoundation#1950)

codis/config/dashboard.toml

@@ -33,9 +33,10 @@ migration_async_numkeys = 500
 migration_timeout = "30s"
 
 # Set configs for redis sentinel.
-sentinel_check_server_state_interval = "5s"
-sentinel_check_master_failover_interval = "1s"
-sentinel_master_dead_check_times = 5
+sentinel_check_server_state_interval = "10s"
+sentinel_check_master_failover_interval = "2s"
+sentinel_master_dead_check_times = 10
+sentinel_check_offline_server_interval = "2s"
 sentinel_client_timeout = "10s"
 sentinel_quorum = 2
 sentinel_parallel_syncs = 1

codis/pkg/models/action.go

@@ -11,4 +11,7 @@ const (
 	ActionMigrating = "migrating"
 	ActionFinished  = "finished"
 	ActionSyncing   = "syncing"
+	ActionSynced    = "synced"
+
+	ActionSyncedFailed = "synced_failed"
 )

codis/pkg/models/group.go

@@ -33,6 +33,13 @@ const (
 	GroupServerStateOffline
 )
 
+type GroupServerRole string
+
+const (
+	RoleMaster GroupServerRole = "master"
+	RoleSlave  GroupServerRole = "slave"
+)
+
 type GroupServer struct {
 	Addr       string `json:"server"`
 	DataCenter string `json:"datacenter"`
@@ -43,9 +50,11 @@ type GroupServer struct {
 	} `json:"action"`
 
 	// master or slave
-	Role string `json:"role"`
+	Role GroupServerRole `json:"role"`
 	// If it is a master node, take the master_repl_offset field, otherwise take the slave_repl_offset field
-	ReplyOffset int `json:"reply_offset"`
+	DbBinlogFileNum uint64 `json:"binlog_file_num"` // db0
+	DbBinlogOffset  uint64 `json:"binlog_offset"`   // db0
+
 	// Monitoring status, 0 normal, 1 subjective offline, 2 actual offline
 	// If marked as 2 , no service is provided
 	State GroupServerState `json:"state"`

codis/pkg/topom/config.go

@@ -50,9 +50,10 @@ migration_async_numkeys = 500
 migration_timeout = "30s"
 
 # Set configs for redis sentinel.
-sentinel_check_server_state_interval = "5s"
-sentinel_check_master_failover_interval = "1s"
-sentinel_master_dead_check_times = 5
+sentinel_check_server_state_interval = "10s"
+sentinel_check_master_failover_interval = "2s"
+sentinel_master_dead_check_times = 10
+sentinel_check_offline_server_interval = "2s"
 sentinel_client_timeout = "10s"
 sentinel_quorum = 2
 sentinel_parallel_syncs = 1
@@ -86,6 +87,7 @@ type Config struct {
 	SentinelCheckServerStateInterval    timesize.Duration `toml:"sentinel_check_server_state_interval" json:"sentinel_client_timeout"`
 	SentinelCheckMasterFailoverInterval timesize.Duration `toml:"sentinel_check_master_failover_interval" json:"sentinel_check_master_failover_interval"`
 	SentinelMasterDeadCheckTimes        int8              `toml:"sentinel_master_dead_check_times" json:"sentinel_master_dead_check_times"`
+	SentinelCheckOfflineServerInterval  timesize.Duration `toml:"sentinel_check_offline_server_interval" json:"sentinel_check_offline_server_interval"`
 	SentinelClientTimeout               timesize.Duration `toml:"sentinel_client_timeout" json:"sentinel_client_timeout"`
 	SentinelQuorum                      int               `toml:"sentinel_quorum" json:"sentinel_quorum"`
 	SentinelParallelSyncs               int               `toml:"sentinel_parallel_syncs" json:"sentinel_parallel_syncs"`

codis/pkg/topom/topom.go

@@ -210,12 +210,12 @@ func (s *Topom) Start(routines bool) error {
 		}
 	}, nil, true, 0)
 
-	// Check the status of the pre-offline master every 1 second
+	// Check the status of the pre-offline master every 2 second
 	// to determine whether to automatically switch master and slave
 	gxruntime.GoUnterminated(func() {
 		for !s.IsClosed() {
 			if s.IsOnline() {
-				w, _ := s.CheckPreOffineMastersState(5 * time.Second)
+				w, _ := s.CheckPreOfflineMastersState(5 * time.Second)
 				if w != nil {
 					w.Wait()
 				}
@@ -224,6 +224,20 @@ func (s *Topom) Start(routines bool) error {
 		}
 	}, nil, true, 0)
 
+	// Check the status of the offline master and slave every 30 second
+	// to determine whether to automatically recover to right master-slave replication relationship
+	gxruntime.GoUnterminated(func() {
+		for !s.IsClosed() {
+			if s.IsOnline() {
+				w, _ := s.CheckOfflineMastersAndSlavesState(5 * time.Second)
+				if w != nil {
+					w.Wait()
+				}
+			}
+			time.Sleep(s.Config().SentinelCheckOfflineServerInterval.Duration())
+		}
+	}, nil, true, 0)
+
 	gxruntime.GoUnterminated(func() {
 		for !s.IsClosed() {
 			if s.IsOnline() {