Problem: dynamic-level-bytes is not enabled for restored application.db

CHANGELOG.md

@@ -23,6 +23,7 @@
 ### Improvements
 
 - [#890](https://github.com/crypto-org-chain/cronos/pull/890) optimize memiavl snapshot format.
+- [#904](https://github.com/crypto-org-chain/cronos/pull/904) Enable "dynamic-level-bytes" on new `application.db`.
 
 *Feb 09, 2022*
 

cmd/cronosd/cmd/versiondb.go

@@ -9,6 +9,7 @@ import (
 	"github.com/crypto-org-chain/cronos/v2/app"
 	"github.com/crypto-org-chain/cronos/v2/cmd/cronosd/opendb"
 	versiondbclient "github.com/crypto-org-chain/cronos/versiondb/client"
+	"github.com/linxGnu/grocksdb"
 	"github.com/spf13/cobra"
 )
 
@@ -21,8 +22,10 @@ func ChangeSetCmd() *cobra.Command {
 	sort.Strings(storeNames)
 
 	return versiondbclient.ChangeSetGroupCmd(versiondbclient.Options{
-		DefaultStores:     storeNames,
-		OpenReadOnlyDB:    opendb.OpenReadOnlyDB,
-		AppRocksDBOptions: opendb.NewRocksdbOptions,
+		DefaultStores:  storeNames,
+		OpenReadOnlyDB: opendb.OpenReadOnlyDB,
+		AppRocksDBOptions: func(sstFileWriter bool) *grocksdb.Options {
+			return opendb.NewRocksdbOptions(nil, sstFileWriter)
+		},
 	})
 }

cmd/cronosd/opendb/opendb_rocksdb.go

@@ -6,24 +6,18 @@ package opendb
 import (
 	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/linxGnu/grocksdb"
 	dbm "github.com/tendermint/tm-db"
 )
 
+const BlockCacheSize = 1 << 30
+
 func OpenDB(home string, backendType dbm.BackendType) (dbm.DB, error) {
 	dataDir := filepath.Join(home, "data")
 	if backendType == dbm.RocksDBBackend {
-		// customize rocksdb options
-		db, err := grocksdb.OpenDb(NewRocksdbOptions(false), filepath.Join(dataDir, "application.db"))
-		if err != nil {
-			return nil, err
-		}
-		ro := grocksdb.NewDefaultReadOptions()
-		wo := grocksdb.NewDefaultWriteOptions()
-		woSync := grocksdb.NewDefaultWriteOptions()
-		woSync.SetSync(true)
-		return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+		return openRocksdb(filepath.Join(dataDir, "application.db"), false)
 	}
 
 	return dbm.NewDB("application", backendType, dataDir)
@@ -33,24 +27,58 @@ func OpenDB(home string, backendType dbm.BackendType) (dbm.DB, error) {
 func OpenReadOnlyDB(home string, backendType dbm.BackendType) (dbm.DB, error) {
 	dataDir := filepath.Join(home, "data")
 	if backendType == dbm.RocksDBBackend {
-		// customize rocksdb options
-		db, err := grocksdb.OpenDbForReadOnly(NewRocksdbOptions(false), filepath.Join(dataDir, "application.db"), false)
-		if err != nil {
-			return nil, err
-		}
-
-		ro := grocksdb.NewDefaultReadOptions()
-		wo := grocksdb.NewDefaultWriteOptions()
-		woSync := grocksdb.NewDefaultWriteOptions()
-		woSync.SetSync(true)
-		return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+		return openRocksdb(filepath.Join(dataDir, "application.db"), true)
 	}
 
 	return dbm.NewDB("application", backendType, dataDir)
 }
 
-func NewRocksdbOptions(sstFileWriter bool) *grocksdb.Options {
-	opts := grocksdb.NewDefaultOptions()
+func openRocksdb(dir string, readonly bool) (dbm.DB, error) {
+	opts, err := loadLatestOptions(dir)
+	if err != nil {
+		return nil, err
+	}
+	// customize rocksdb options
+	opts = NewRocksdbOptions(opts, false)
+
+	var db *grocksdb.DB
+	if readonly {
+		db, err = grocksdb.OpenDbForReadOnly(opts, dir, false)
+	} else {
+		db, err = grocksdb.OpenDb(opts, dir)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	ro := grocksdb.NewDefaultReadOptions()
+	wo := grocksdb.NewDefaultWriteOptions()
+	woSync := grocksdb.NewDefaultWriteOptions()
+	woSync.SetSync(true)
+	return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+}
+
+// loadLatestOptions try to load options from existing db, returns nil if not exists.
+func loadLatestOptions(dir string) (*grocksdb.Options, error) {
+	opts, err := grocksdb.LoadLatestOptions(dir, grocksdb.NewDefaultEnv(), true, grocksdb.NewLRUCache(BlockCacheSize))
+	if err != nil {
+		// not found is not an error
+		if strings.HasPrefix(err.Error(), "NotFound: ") {
+			return nil, nil
+		}
+		return nil, err
+	}
+	return opts.Options(), nil
+}
+
+// NewRocksdbOptions build options for `application.db`,
+// it overrides existing options if provided, otherwise create new one assuming it's a new database.
+func NewRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool) *grocksdb.Options {
+	if opts == nil {
+		opts = grocksdb.NewDefaultOptions()
+		// only enable dynamic-level-bytes on new db, don't override for existing db
+		opts.SetLevelCompactionDynamicLevelBytes(true)
+	}
 	opts.SetCreateIfMissing(true)
 	opts.IncreaseParallelism(runtime.NumCPU())
 	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
@@ -60,7 +88,7 @@ func NewRocksdbOptions(sstFileWriter bool) *grocksdb.Options {
 	bbto := grocksdb.NewDefaultBlockBasedTableOptions()
 
 	// 1G block cache
-	bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30))
+	bbto.SetBlockCache(grocksdb.NewLRUCache(BlockCacheSize))
 
 	// http://rocksdb.org/blog/2021/12/29/ribbon-filter.html
 	bbto.SetFilterPolicy(grocksdb.NewRibbonHybridFilterPolicy(9.9, 1))

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
-	github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b
+	github.com/linxGnu/grocksdb v1.7.15
 	github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974
 	github.com/rakyll/statik v0.1.7
 	github.com/spf13/cast v1.5.0
@@ -215,7 +215,7 @@ replace (
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.7.0
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	// use replace to force update grocksdb dependency in tm-db
-	github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b
+	github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.7.16-0.20230303082518-587b6c4f0dab
 	github.com/miguelmota/go-ethereum-hdwallet => github.com/crypto-org-chain/go-ethereum-hdwallet v0.1.2
 
 	github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20230126051749-d984b1562242

go.sum

@@ -855,8 +855,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b h1:Olh1b4gdN5J9ZZyCiudSZ7HLrU+IL4SkYFcgyM9rGsA=
-github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
+github.com/linxGnu/grocksdb v1.7.16-0.20230303082518-587b6c4f0dab h1:aTVNLqaxBD1ue7MqBEU5rkeVd8LyBJxzpf3EtkDEGiI=
+github.com/linxGnu/grocksdb v1.7.16-0.20230303082518-587b6c4f0dab/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
 github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=

gomod2nix.toml

@@ -343,8 +343,8 @@ schema = 3
     version = "v0.1.0"
     hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ="
   [mod."github.com/linxGnu/grocksdb"]
-    version = "v1.7.15-0.20230222024938-b61261a9193b"
-    hash = "sha256-Hry5mpO8WqCuYZ0zCnOt6kopDGprc7/nI318A2D+Kk0="
+    version = "v1.7.16-0.20230303082518-587b6c4f0dab"
+    hash = "sha256-MuH+rIx+r3zirXdKPHFRYeAP7QORr9gSwSiCJST7534="
     replaced = "github.com/linxGnu/grocksdb"
   [mod."github.com/magiconair/properties"]
     version = "v1.8.6"

versiondb/go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/gogo/protobuf v1.3.3
 	github.com/golang/snappy v0.0.4
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/linxGnu/grocksdb v1.7.14
+	github.com/linxGnu/grocksdb v1.7.15
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.1

versiondb/go.sum

@@ -614,8 +614,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.7.14 h1:8lMZzyWeNP5lI0BIppX05DzmQzXj/Tgu82bgWYtowLY=
-github.com/linxGnu/grocksdb v1.7.14/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
+github.com/linxGnu/grocksdb v1.7.15 h1:AEhP28lkeAybv5UYNYviYISpR6bJejEnKuYbnWAnxx0=
+github.com/linxGnu/grocksdb v1.7.15/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
 github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=