Problem: version mismatch happen occasionally

CHANGELOG.md

@@ -9,6 +9,7 @@
 * [#1724](https://github.com/crypto-org-chain/cronos/pull/1724) Include the fix of nonce management in batch tx in ethermint.
 * [#1748](https://github.com/crypto-org-chain/cronos/pull/1748) Query with GetCFWithTS to compare both timestamp and key to avoid run fixdata multiple times.
 * (versiondb) [#1751](https://github.com/crypto-org-chain/cronos/pull/1751) Add missing Destroy for read options to properly hold and release options reference.
+* [#1759](https://github.com/crypto-org-chain/cronos/pull/1759) Fix version mismatch happen occasionally.
 
 ### Improvements
 

app/app.go

@@ -1041,16 +1041,28 @@ func New(
 	}
 
 	if loadLatest {
-		if err := app.LoadLatestVersion(); err != nil {
-			tmos.Exit(err.Error())
+		var qmsVersion int64
+		if app.qms != nil {
+			qmsVersion = app.qms.LatestVersion()
 		}
 
-		if app.qms != nil {
-			v1 := app.qms.LatestVersion()
-			v2 := app.LastBlockHeight()
-			if v1 > 0 && v1 < v2 {
+		if qmsVersion == 0 {
+			if err := app.LoadLatestVersion(); err != nil {
+				tmos.Exit(err.Error())
+			}
+		} else {
+			// make sure iavl version is not ahead of versiondb version
+			app.SetStoreLoader(VersionStoreLoader(qmsVersion))
+
+			if err := app.LoadLatestVersion(); err != nil {
+				tmos.Exit(err.Error())
+			}
+
+			// still keep the check for safety
+			iavlVersion := app.LastBlockHeight()
+			if qmsVersion < iavlVersion {
 				// try to prevent gap being created in versiondb
-				tmos.Exit(fmt.Sprintf("versiondb version %d lag behind iavl version %d", v1, v2))
+				tmos.Exit(fmt.Sprintf("versiondb version %d lag behind iavl version %d", qmsVersion, iavlVersion))
 			}
 		}
 
@@ -1491,3 +1503,10 @@ func (app *App) CheckTx(req *abci.RequestCheckTx) (*abci.ResponseCheckTx, error)
 
 	return app.BaseApp.CheckTx(req)
 }
+
+// VersionStoreLoader will be used when there's versiondb
+func VersionStoreLoader(version int64) baseapp.StoreLoader {
+	return func(ms storetypes.CommitMultiStore) error {
+		return ms.LoadVersion(version)
+	}
+}

memiavl/db.go

@@ -560,9 +560,18 @@
 			if err != nil {
 				return 0, err
 			}
-			if err := db.wal.Write(entry.index, bz); err != nil {
+
+			lastIndex, err := db.wal.LastIndex()
+			if err != nil {
 				return 0, err
 			}
+			if entry.index < lastIndex+1 {
+				db.logger.Error("commit old version idempotently", "expected", lastIndex+1, "actual", entry.index)
+			} else {
+				if err := db.wal.Write(entry.index, bz); err != nil {
+					return 0, err
+				}
+			}
 		}
 	}
 
@@ -591,13 +600,25 @@
 				break
 			}
 
-			for _, entry := range entries {
+			lastIndex, err := db.wal.LastIndex()
+			if err != nil {
+				walQuit <- err
+				return
+			}
+
+			for i, entry := range entries {
 				bz, err := entry.data.Marshal()
 				if err != nil {
 					walQuit <- err
 					return
 				}
-				batch.Write(entry.index, bz)
+
+				if entry.index < lastIndex+uint64(i+1) {
+					db.logger.Error("commit old version idempotently", "expected", lastIndex+uint64(i+1), "actual", entry.index)
+					continue
+				} else {
+					batch.Write(entry.index, bz)
+				}
 			}
 
 			if err := db.wal.WriteBatch(&batch); err != nil {