Merge pull request sorintlab#529 from gesslein/patch-1

Fix FAQ link to architecture.md
sgotti · Jul 2, 2018 · 07bdac8 · 07bdac8
2 parents e81e5d7 + a93d428
commit 07bdac8
Show file tree

Hide file tree

Showing 9 changed files with 679 additions and 29 deletions.
diff --git a/cmd/keeper/cmd/keeper.go b/cmd/keeper/cmd/keeper.go
@@ -611,6 +611,22 @@ func parseSynchronousStandbyNames(s string) ([]string, error) {
 	return entries, nil
 }
 
+func (p *PostgresKeeper) GetInSyncStandbys() ([]string, error) {
+	inSyncStandbysFullName, err := p.pgm.GetSyncStandbys()
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve current sync standbys status from instance: %v", err)
+	}
+
+	inSyncStandbys := []string{}
+	for _, s := range inSyncStandbysFullName {
+		if common.IsStolonName(s) {
+			inSyncStandbys = append(inSyncStandbys, common.NameFromStolonName(s))
+		}
+	}
+
+	return inSyncStandbys, nil
+}
+
 func (p *PostgresKeeper) GetPGState(pctx context.Context) (*cluster.PostgresState, error) {
 	p.getPGStateMutex.Lock()
 	defer p.getPGStateMutex.Unlock()
@@ -644,20 +660,13 @@ func (p *PostgresKeeper) GetPGState(pctx context.Context) (*cluster.PostgresStat
 		log.Debugw("filtered out managed pg parameters", "filteredPGParameters", filteredPGParameters)
 		pgState.PGParameters = filteredPGParameters
 
-		synchronousStandbyNames, err := parseSynchronousStandbyNames(pgParameters["synchronous_standby_names"])
+		inSyncStandbys, err := p.GetInSyncStandbys()
 		if err != nil {
-			log.Errorw("error parsing synchronous_standby_names", zap.Error(err))
+			log.Errorw("failed to retrieve current in sync standbys from instance", zap.Error(err))
 			return pgState, nil
 		}
-		synchronousStandbys := []string{}
-		for _, n := range synchronousStandbyNames {
-			// pgState.SynchronousStandbys must contain only the internal standbys dbUIDs
-			if !common.IsStolonName(n) {
-				continue
-			}
-			synchronousStandbys = append(synchronousStandbys, common.NameFromStolonName(n))
-		}
-		pgState.SynchronousStandbys = synchronousStandbys
+
+		pgState.SynchronousStandbys = inSyncStandbys
 
 		sd, err := p.pgm.GetSystemData()
 		if err != nil {

diff --git a/cmd/sentinel/cmd/sentinel.go b/cmd/sentinel/cmd/sentinel.go
@@ -306,7 +306,7 @@ func (s *Sentinel) updateKeepersStatus(cd *cluster.ClusterData, keepersInfo clus
 			db.Status.TimelinesHistory = dbs.TimelinesHistory
 			db.Status.PGParameters = cluster.PGParameters(dbs.PGParameters)
 
-			db.Status.SynchronousStandbys = dbs.SynchronousStandbys
+			db.Status.CurSynchronousStandbys = dbs.SynchronousStandbys
 
 			db.Status.OlderWalFile = dbs.OlderWalFile
 		} else {
@@ -1202,13 +1202,50 @@ func (s *Sentinel) updateCluster(cd *cluster.ClusterData, pis cluster.ProxiesInf
 						}
 					}
 
+					// if the current known in sync syncstandbys are different than the required ones wait for them and remove non good ones
+					if !util.CompareStringSliceNoOrder(masterDB.Status.SynchronousStandbys, masterDB.Spec.SynchronousStandbys) {
+
+						// remove old syncstandbys from current status
+						masterDB.Status.SynchronousStandbys = util.CommonElements(masterDB.Status.SynchronousStandbys, masterDB.Spec.SynchronousStandbys)
+
+						// add reported in sync syncstandbys to the current status
+						curSyncStandbys := util.CommonElements(masterDB.Status.CurSynchronousStandbys, masterDB.Spec.SynchronousStandbys)
+						toAddSyncStandbys := util.Difference(curSyncStandbys, masterDB.Status.SynchronousStandbys)
+						masterDB.Status.SynchronousStandbys = append(masterDB.Status.SynchronousStandbys, toAddSyncStandbys...)
+
+						// if some of the non yet in sync syncstandbys are failed, set Spec.SynchronousStandbys to the current in sync ones, se other could be added.
+						notInSyncSyncStandbys := util.Difference(masterDB.Spec.SynchronousStandbys, masterDB.Status.SynchronousStandbys)
+						update := false
+						for _, dbUID := range notInSyncSyncStandbys {
+							if _, ok := newcd.DBs[dbUID]; !ok {
+								log.Infow("one of the new synchronousStandbys has been removed", "db", dbUID, "inSyncStandbys", masterDB.Status.SynchronousStandbys, "synchronousStandbys", masterDB.Spec.SynchronousStandbys)
+								update = true
+								continue
+							}
+							if _, ok := goodStandbys[dbUID]; !ok {
+								log.Infow("one of the new synchronousStandbys is not in good state", "db", dbUID, "inSyncStandbys", masterDB.Status.SynchronousStandbys, "synchronousStandbys", masterDB.Spec.SynchronousStandbys)
+								update = true
+								continue
+							}
+						}
+						if update {
+							// Use the current known in sync syncStandbys as Spec.SynchronousStandbys
+							log.Infow("setting the expected sync-standbys to the current known in sync sync-standbys", "inSyncStandbys", masterDB.Status.SynchronousStandbys, "synchronousStandbys", masterDB.Spec.SynchronousStandbys)
+							masterDB.Spec.SynchronousStandbys = masterDB.Status.SynchronousStandbys
+
+							// Just sort to always have them in the same order and avoid
+							// unneeded updates to synchronous_standby_names by the keeper.
+							sort.Sort(sort.StringSlice(masterDB.Spec.SynchronousStandbys))
+						}
+					}
+
 					// update synchronousStandbys only if the reported
 					// SynchronousStandbys are the same as the required ones. In
 					// this way, when we have to choose a new master we are sure
 					// that there're no intermediate changes between the
 					// reported standbys and the required ones.
 					if !util.CompareStringSliceNoOrder(masterDB.Status.SynchronousStandbys, masterDB.Spec.SynchronousStandbys) {
-						log.Infof("won't update masterDB required synchronous standby since the latest master reported synchronous standbys are different from the db spec ones", "reported", curMasterDB.Status.SynchronousStandbys, "spec", curMasterDB.Spec.SynchronousStandbys)
+						log.Infow("waiting for new defined synchronous standbys to be in sync", "inSyncStandbys", curMasterDB.Status.SynchronousStandbys, "synchronousStandbys", curMasterDB.Spec.SynchronousStandbys)
 					} else {
 						addFakeStandby := false
 						externalSynchronousStandbys := map[string]struct{}{}
@@ -1358,6 +1395,9 @@ func (s *Sentinel) updateCluster(cd *cluster.ClusterData, pis cluster.ProxiesInf
 							masterDB.Spec.ExternalSynchronousStandbys = append(masterDB.Spec.ExternalSynchronousStandbys, fakeStandbyName)
 						}
 
+						// remove old syncstandbys from current status
+						masterDB.Status.SynchronousStandbys = util.CommonElements(masterDB.Status.SynchronousStandbys, masterDB.Spec.SynchronousStandbys)
+
 						// Just sort to always have them in the same order and avoid
 						// unneeded updates to synchronous_standby_names by the keeper.
 						sort.Sort(sort.StringSlice(masterDB.Spec.SynchronousStandbys))
@@ -1367,6 +1407,8 @@ func (s *Sentinel) updateCluster(cd *cluster.ClusterData, pis cluster.ProxiesInf
 					masterDB.Spec.SynchronousReplication = false
 					masterDB.Spec.SynchronousStandbys = nil
 					masterDB.Spec.ExternalSynchronousStandbys = nil
+
+					masterDB.Status.SynchronousStandbys = nil
 				}
 
 				// NotFailed != Good since there can be some dbs that are converging