NRG (2.11): Always write term/vote and peer state synchronously (#5848)

Signed-off-by: Neil Twigg <neil@nats.io>

server/filestore.go

@@ -9834,14 +9834,22 @@ func (alg StoreCompression) Decompress(buf []byte) ([]byte, error) {
 // sets O_SYNC on the open file if SyncAlways is set. The dios semaphore is
 // handled automatically by this function, so don't wrap calls to it in dios.
 func (fs *fileStore) writeFileWithOptionalSync(name string, data []byte, perm fs.FileMode) error {
+	if fs.fcfg.SyncAlways {
+		return writeFileWithSync(name, data, perm)
+	}
 	<-dios
 	defer func() {
 		dios <- struct{}{}
 	}()
-	flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
-	if fs.fcfg.SyncAlways {
-		flags |= os.O_SYNC
-	}
+	return os.WriteFile(name, data, perm)
+}
+
+func writeFileWithSync(name string, data []byte, perm fs.FileMode) error {
+	<-dios
+	defer func() {
+		dios <- struct{}{}
+	}()
+	flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_SYNC
 	f, err := os.OpenFile(name, flags, perm)
 	if err != nil {
 		return err

server/raft.go

@@ -1165,11 +1165,7 @@ func (n *raft) installSnapshot(snap *snapshot) error {
 	sn := fmt.Sprintf(snapFileT, snap.lastTerm, snap.lastIndex)
 	sfile := filepath.Join(snapDir, sn)
 
-	<-dios
-	err := os.WriteFile(sfile, n.encodeSnapshot(snap), defaultFilePerms)
-	dios <- struct{}{}
-
-	if err != nil {
+	if err := writeFileWithSync(sfile, n.encodeSnapshot(snap), defaultFilePerms); err != nil {
 		// We could set write err here, but if this is a temporary situation, too many open files etc.
 		// we want to retry and snapshots are not fatal.
 		return err
@@ -3874,12 +3870,7 @@ func writePeerState(sd string, ps *peerState) error {
 	if _, err := os.Stat(psf); err != nil && !os.IsNotExist(err) {
 		return err
 	}
-
-	<-dios
-	err := os.WriteFile(psf, encodePeerState(ps), defaultFilePerms)
-	dios <- struct{}{}
-
-	return err
+	return writeFileWithSync(psf, encodePeerState(ps), defaultFilePerms)
 }
 
 func readPeerState(sd string) (ps *peerState, err error) {
@@ -3903,12 +3894,7 @@ func writeTermVote(sd string, wtv []byte) error {
 	if _, err := os.Stat(psf); err != nil && !os.IsNotExist(err) {
 		return err
 	}
-
-	<-dios
-	err := os.WriteFile(psf, wtv, defaultFilePerms)
-	dios <- struct{}{}
-
-	return err
+	return writeFileWithSync(psf, wtv, defaultFilePerms)
 }
 
 // readTermVote will read the largest term and who we voted from to stable storage.